From 993044908d29bf5c6b6b56f11b748043e21f4bd2 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Mon, 8 Aug 2022 18:40:51 +0100 Subject: [PATCH 01/11] Add environment handling and context manager --- pulp/apis/gurobi_api.py | 105 ++++++++++++++++++++++++++------ pulp/tests/test_gurobipy_env.py | 36 +++++++++++ 2 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 pulp/tests/test_gurobipy_env.py diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 6e2210e5..d004feed 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -33,7 +33,7 @@ import warnings # to import the gurobipy name into the module scope -gurobipy = None +gp = None class GUROBI(LpSolver): @@ -46,12 +46,13 @@ class GUROBI(LpSolver): """ name = "GUROBI" + env = None try: sys.path.append(gurobi_path) # to import the name into the module scope - global gurobipy - import gurobipy + global gp + import gurobipy as gp except: # FIXME: Bug because gurobi returns # a gurobi exception on failed imports def available(self): @@ -73,7 +74,10 @@ def __init__( gapRel=None, warmStart=False, logPath=None, - **solverParams + env=None, + envOptions=None, + manageEnv=True, + **solverParams, ): """ :param bool mip: if False, assume LP even if integer variables @@ -83,13 +87,24 @@ def __init__( :param bool warmStart: if True, the solver will use the current value of variables as a start :param str logPath: path to the log file :param float epgap: deprecated for gapRel + :param gp.Env env: gurobipyEnv. Default None. + :param dict envOptions: environment options + :param bool manageEnv: if False, assume gp.Env's is handled + internally and outside of context manager """ + self.env = env + self.env_options = envOptions if envOptions else {} + self.manage_env = manageEnv + self.model = None + self.init = True + if epgap is not None: warnings.warn("Parameter epgap is being depreciated for gapRel") if gapRel is not None: warnings.warn("Parameter gapRel and epgap passed, using gapRel") else: gapRel = epgap + LpSolver.__init__( self, mip=mip, @@ -99,18 +114,38 @@ def __init__( logPath=logPath, warmStart=warmStart, ) + # set the output of gurobi if not self.msg: - gurobipy.setParam("OutputFlag", 0) + self.env_options["OutputFlag"] = 0 - # set the gurobi parameter values + # Append all parameters to env_options for key, value in solverParams.items(): - gurobipy.setParam(key, value) + self.env_options[key] = value + + def __enter__(self): + try: + self.env = gp.Env(params=self.env_options) + self.model = gp.Model(env=self.env) + # No need to manage_env outside of context manager + self.manage_env = False + except gp.GurobiError as e: + raise e + return self + + def __exit__(self, type, value, tb): + self.model.dispose() + self.env.dispose() + + def __del__(self): + if self.manage_env: + self.model.dispose() + self.env.dispose() def findSolutionValues(self, lp): model = lp.solverModel solutionStatus = model.Status - GRB = gurobipy.GRB + GRB = gp.GRB # TODO: check status for Integer Feasible gurobiLpStatus = { GRB.OPTIMAL: constants.LpStatusOptimal, @@ -164,13 +199,41 @@ def findSolutionValues(self, lp): def available(self): """True if the solver is available""" + self.initGurobi() + m = self.model.copy() + m.setParam("OutputFlag", 0) try: - gurobipy.setParam("_test", 0) - except gurobipy.GurobiError as e: + m.addVars(range(2001)) + m.setParam("OutputFlag", 0) + m.optimize() + except gp.GurobiError as e: warnings.warn("GUROBI error: {}.".format(e)) return False + finally: + m.dispose() + # If user is handling environment, make sure we still set parameters + if not self.manage_env and self.env_options: + for param, value in self.env_options: + self.model.setParam(param, value) return True + def initGurobi(self): + # Environment handled here + if self.manage_env and self.init and not self.env and not self.model: + try: + self.env = gp.Env(params=self.env_options) + self.model = gp.Model(env=self.env) + self.init = False + except gp.GurobiError as e: + raise e + # Environment handled by user + elif not self.manage_env and self.init and self.env and not self.model: + try: + self.model = gp.Model(env=self.env) + self.init = False + except gp.GurobiError as e: + raise e + def callSolver(self, lp, callback=None): """Solves the problem with gurobi""" # solve the problem @@ -183,7 +246,9 @@ def buildSolverModel(self, lp): Takes the pulp lp model and translates it into a gurobi model """ log.debug("create the gurobi model") - lp.solverModel = gurobipy.Model(lp.name) + self.initGurobi() + self.model.ModelName = lp.name + lp.solverModel = self.model log.debug("set the sense of the problem") if lp.sense == constants.LpMaximize: lp.solverModel.setAttr("ModelSense", -1) @@ -200,14 +265,14 @@ def buildSolverModel(self, lp): for var in lp.variables(): lowBound = var.lowBound if lowBound is None: - lowBound = -gurobipy.GRB.INFINITY + lowBound = -gp.GRB.INFINITY upBound = var.upBound if upBound is None: - upBound = gurobipy.GRB.INFINITY + upBound = gp.GRB.INFINITY obj = lp.objective.get(var, 0.0) - varType = gurobipy.GRB.CONTINUOUS + varType = gp.GRB.CONTINUOUS if var.cat == constants.LpInteger and self.mip: - varType = gurobipy.GRB.INTEGER + varType = gp.GRB.INTEGER var.solverVar = lp.solverModel.addVar( lowBound, upBound, vtype=varType, obj=obj, name=var.name ) @@ -222,15 +287,15 @@ def buildSolverModel(self, lp): log.debug("add the Constraints to the problem") for name, constraint in lp.constraints.items(): # build the expression - expr = gurobipy.LinExpr( + expr = gp.LinExpr( list(constraint.values()), [v.solverVar for v in constraint.keys()] ) if constraint.sense == constants.LpConstraintLE: - relation = gurobipy.GRB.LESS_EQUAL + relation = gp.GRB.LESS_EQUAL elif constraint.sense == constants.LpConstraintGE: - relation = gurobipy.GRB.GREATER_EQUAL + relation = gp.GRB.GREATER_EQUAL elif constraint.sense == constants.LpConstraintEQ: - relation = gurobipy.GRB.EQUAL + relation = gp.GRB.EQUAL else: raise PulpSolverError("Detected an invalid constraint type") constraint.solverConstraint = lp.solverModel.addConstr( @@ -267,7 +332,7 @@ def actualResolve(self, lp, callback=None): for constraint in lp.constraints.values(): if constraint.modified: constraint.solverConstraint.setAttr( - gurobipy.GRB.Attr.RHS, -constraint.constant + gp.GRB.Attr.RHS, -constraint.constant ) lp.solverModel.update() self.callSolver(lp, callback=callback) diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py new file mode 100644 index 00000000..6d661e5a --- /dev/null +++ b/pulp/tests/test_gurobipy_env.py @@ -0,0 +1,36 @@ +import unittest + +from pulp import GUROBI, LpProblem, LpVariable, const + + +class GurobiEnvTests(unittest.TestCase): + def setUp(self): + self.options = {"OutputFlag": 1, "MemLimit": 1} + self.prob = LpProblem("test011", const.LpMaximize) + x = LpVariable("x", 0, 1) + y = LpVariable("y", 0, 1) + z = LpVariable("z", 0, 1) + self.prob += x + y + z, "obj" + self.prob += x + y + z <= 1, "c1" + + def testContextManager(self): + # Using solver within a context manager + with GUROBI(msg=True, **self.options) as solver: + status = self.prob.solve(solver) + + def testGpEnv(self): + # Using gp.Env within a context manager + import gurobipy as gp + + with gp.Env() as env: + solver = GUROBI(msg=True, manageEnv=False, env=env, **self.options) + self.prob.solve(solver) + + def testDefault(self): + solver = GUROBI(msg=True, **self.options) + self.prob.solve(solver) + + def testNoEnv(self): + # Failing test for no environment handling + solver = GUROBI(msg=True, manageEnv=False, envOptions=self.options) + self.assertRaises(AttributeError, self.prob.solve, solver) From f08b8bba0f12373a745b2c8d8a9cef5aac7b8953 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Tue, 4 Oct 2022 00:10:26 +1100 Subject: [PATCH 02/11] Dispose of solvers in listSolvers --- pulp/apis/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pulp/apis/__init__.py b/pulp/apis/__init__.py index c1067451..b8390dec 100644 --- a/pulp/apis/__init__.py +++ b/pulp/apis/__init__.py @@ -142,10 +142,13 @@ def listSolvers(onlyAvailable=False): :return: list of solver names :rtype: list """ - solvers = [s() for s in _all_solvers] - if onlyAvailable: - return [solver.name for solver in solvers if solver.available()] - return [solver.name for solver in solvers] + result = [] + for s in _all_solvers: + solver = s() + if (not onlyAvailable) or solver.available(): + result.append(solver.name) + del solver + return result # DEPRECATED aliases: From 691e7e9ca38b19c3d76244a3605c5ad64e9cb718 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Tue, 4 Oct 2022 00:22:05 +1100 Subject: [PATCH 03/11] Close environment in Gurobi availability check --- pulp/apis/gurobi_api.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index d004feed..4d33e35c 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -199,22 +199,12 @@ def findSolutionValues(self, lp): def available(self): """True if the solver is available""" - self.initGurobi() - m = self.model.copy() - m.setParam("OutputFlag", 0) try: - m.addVars(range(2001)) - m.setParam("OutputFlag", 0) - m.optimize() - except gp.GurobiError as e: + with gp.Env(): + pass + except gurobipy.GurobiError as e: warnings.warn("GUROBI error: {}.".format(e)) return False - finally: - m.dispose() - # If user is handling environment, make sure we still set parameters - if not self.manage_env and self.env_options: - for param, value in self.env_options: - self.model.setParam(param, value) return True def initGurobi(self): From 70c156d301ba73279150ed0c87b6e8e96b6ea6fe Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Mon, 8 Aug 2022 18:40:51 +0100 Subject: [PATCH 04/11] Add environment handling and context manager --- pulp/apis/gurobi_api.py | 114 +++++++++++++++---------------- pulp/tests/test_gurobipy_env.py | 116 ++++++++++++++++++++++++++------ 2 files changed, 149 insertions(+), 81 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 4d33e35c..7f51cb05 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -76,7 +76,7 @@ def __init__( logPath=None, env=None, envOptions=None, - manageEnv=True, + manageEnv=False, **solverParams, ): """ @@ -94,9 +94,11 @@ def __init__( """ self.env = env self.env_options = envOptions if envOptions else {} - self.manage_env = manageEnv + self.manage_env = False if self.env is not None else manageEnv + self.solver_params = solverParams + self.model = None - self.init = True + self.init_gurobi = False # whether env and model have been initialised if epgap is not None: warnings.warn("Parameter epgap is being depreciated for gapRel") @@ -117,30 +119,20 @@ def __init__( # set the output of gurobi if not self.msg: - self.env_options["OutputFlag"] = 0 - - # Append all parameters to env_options - for key, value in solverParams.items(): - self.env_options[key] = value - - def __enter__(self): - try: - self.env = gp.Env(params=self.env_options) - self.model = gp.Model(env=self.env) - # No need to manage_env outside of context manager - self.manage_env = False - except gp.GurobiError as e: - raise e - return self - - def __exit__(self, type, value, tb): - self.model.dispose() - self.env.dispose() + if self.manage_env: + self.env_options["OutputFlag"] = 0 + else: + self.solver_params["OutputFlag"] = 0 def __del__(self): + self.close() + + def close(self): + if not self.init_gurobi: + return if self.manage_env: - self.model.dispose() self.env.dispose() + self.model.dispose() def findSolutionValues(self, lp): model = lp.solverModel @@ -166,35 +158,34 @@ def findSolutionValues(self, lp): var.isModified = False status = gurobiLpStatus.get(solutionStatus, constants.LpStatusUndefined) lp.assignStatus(status) - if status != constants.LpStatusOptimal: - return status - - # populate pulp solution values - for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) - ): - var.varValue = value - - # populate pulp constraints slack - for constr, value in zip( - lp.constraints.values(), - model.getAttr(GRB.Attr.Slack, model.getConstrs()), - ): - constr.slack = value - - if not model.getAttr(GRB.Attr.IsMIP): + try: + # populate pulp solution values for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) ): - var.dj = value + var.varValue = value - # put pi and slack variables against the constraints + # populate pulp constraints slack for constr, value in zip( lp.constraints.values(), - model.getAttr(GRB.Attr.Pi, model.getConstrs()), + model.getAttr(GRB.Attr.Slack, model.getConstrs()), ): - constr.pi = value - + constr.slack = value + + if not model.getAttr(GRB.Attr.IsMIP): + for var, value in zip( + lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + ): + var.dj = value + + # put pi and slack variables against the constraints + for constr, value in zip( + lp.constraints.values(), + model.getAttr(GRB.Attr.Pi, model.getConstrs()), + ): + constr.pi = value + except gp.GurobiError as e: + raise e return status def available(self): @@ -208,21 +199,26 @@ def available(self): return True def initGurobi(self): - # Environment handled here - if self.manage_env and self.init and not self.env and not self.model: - try: + if self.init_gurobi: + return + else: + self.init_gurobi = True + try: + if self.manage_env: self.env = gp.Env(params=self.env_options) self.model = gp.Model(env=self.env) - self.init = False - except gp.GurobiError as e: - raise e - # Environment handled by user - elif not self.manage_env and self.init and self.env and not self.model: - try: - self.model = gp.Model(env=self.env) - self.init = False - except gp.GurobiError as e: - raise e + # Environment handled by user or default Env + else: + if self.env is not None: + self.model = gp.Model(env=self.env) + else: + self.model = gp.Model() + # Set solver parameters + for param, value in self.solver_params.items(): + self.model.setParam(param, value) + # for param, value in self.env_options: + except gp.GurobiError as e: + raise e def callSolver(self, lp, callback=None): """Solves the problem with gurobi""" diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index 6d661e5a..9e3b6a29 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -1,36 +1,108 @@ import unittest -from pulp import GUROBI, LpProblem, LpVariable, const +import gurobipy as gp + + +from pulp import GUROBI, GUROBI_CMD, LpProblem, LpVariable, const + + +def check_dummy_env(): + with gp.Env(): + pass + + +def generate_lp() -> LpProblem: + prob = LpProblem("test", const.LpMaximize) + x = LpVariable("x", 0, 1) + y = LpVariable("y", 0, 1) + z = LpVariable("z", 0, 1) + prob += x + y + z, "obj" + prob += x + y + z <= 1, "c1" + return prob class GurobiEnvTests(unittest.TestCase): def setUp(self): - self.options = {"OutputFlag": 1, "MemLimit": 1} - self.prob = LpProblem("test011", const.LpMaximize) - x = LpVariable("x", 0, 1) - y = LpVariable("y", 0, 1) - z = LpVariable("z", 0, 1) - self.prob += x + y + z, "obj" - self.prob += x + y + z <= 1, "c1" - - def testContextManager(self): - # Using solver within a context manager - with GUROBI(msg=True, **self.options) as solver: - status = self.prob.solve(solver) + self.options = {"OutputFlag": 1} + self.env_options = {"MemLimit": 1} def testGpEnv(self): # Using gp.Env within a context manager - import gurobipy as gp + with gp.Env(params=self.env_options) as env: + prob = generate_lp() + solver = GUROBI(msg=True, env=env, **self.options) + prob.solve(solver) + solver.close() + check_dummy_env() + def testMultipleGpEnv(self): + # Using the same env multiple times with gp.Env() as env: - solver = GUROBI(msg=True, manageEnv=False, env=env, **self.options) - self.prob.solve(solver) + solver = GUROBI(msg=True, env=env) + prob = generate_lp() + prob.solve(solver) + solver.close() + + solver2 = GUROBI(msg=True, env=env) + prob2 = generate_lp() + prob2.solve(solver2) + solver2.close() + + check_dummy_env() - def testDefault(self): + @unittest.SkipTest + def testBackwardCompatibility(self): + """ + Backward compatibility check as previously the environment was not being + freed. On a single-use license this passes (fails to initialise a dummy + env). + """ solver = GUROBI(msg=True, **self.options) - self.prob.solve(solver) + prob = generate_lp() + prob.solve(solver) + + self.assertRaises(gp.GurobiError, check_dummy_env) + gp.disposeDefaultEnv() + solver.close() + + def testManageEnvTrue(self): + solver = GUROBI(msg=True, manageEnv=True, **self.options) + prob = generate_lp() + prob.solve(solver) + + solver.close() + check_dummy_env() + + def testMultipleSolves(self): + solver = GUROBI(msg=True, manageEnv=True, **self.options) + prob = generate_lp() + prob.solve(solver) + + solver.close() + check_dummy_env() + + solver2 = GUROBI(msg=True, manageEnv=True, **self.options) + prob.solve(solver2) + + solver2.close() + check_dummy_env() + + @unittest.SkipTest + def testLeak(self): + """ + Check that we cannot initialise environments after a memory leak. On a + single-use license this passes (fails to initialise a dummy env with a + memory leak). + """ + solver = GUROBI(msg=True, **self.options) + prob = generate_lp() + prob.solve(solver) + + tmp = solver.model + solver.close() + + solver2 = GUROBI(msg=True, **self.options) - def testNoEnv(self): - # Failing test for no environment handling - solver = GUROBI(msg=True, manageEnv=False, envOptions=self.options) - self.assertRaises(AttributeError, self.prob.solve, solver) + prob2 = generate_lp() + prob2.solve(solver2) + self.assertRaises(gp.GurobiError, check_dummy_env) From 72af8a1e9ac3647d5a7b19c756c6894a8857067f Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Mon, 3 Oct 2022 17:50:39 +0100 Subject: [PATCH 05/11] [coin-or/pulp/586] Fix and add test. Remove deprecation message for model.addConstr --- pulp/apis/gurobi_api.py | 65 +++++++++++++++++++++-------------------- pulp/tests/test_pulp.py | 10 ++++++- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 7f51cb05..8d1480da 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -158,34 +158,35 @@ def findSolutionValues(self, lp): var.isModified = False status = gurobiLpStatus.get(solutionStatus, constants.LpStatusUndefined) lp.assignStatus(status) - try: - # populate pulp solution values - for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) - ): - var.varValue = value - - # populate pulp constraints slack - for constr, value in zip( - lp.constraints.values(), - model.getAttr(GRB.Attr.Slack, model.getConstrs()), - ): - constr.slack = value - - if not model.getAttr(GRB.Attr.IsMIP): + if status in [constants.LpStatusOptimal, constants.LpStatusNotSolved]: + try: + # populate pulp solution values for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) ): - var.dj = value + var.varValue = value - # put pi and slack variables against the constraints + # populate pulp constraints slack for constr, value in zip( lp.constraints.values(), - model.getAttr(GRB.Attr.Pi, model.getConstrs()), + model.getAttr(GRB.Attr.Slack, model.getConstrs()), ): - constr.pi = value - except gp.GurobiError as e: - raise e + constr.slack = value + + if not model.getAttr(GRB.Attr.IsMIP): + for var, value in zip( + lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + ): + var.dj = value + + # put pi and slack variables against the constraints + for constr, value in zip( + lp.constraints.values(), + model.getAttr(GRB.Attr.Pi, model.getConstrs()), + ): + constr.pi = value + except gp.GurobiError as e: + raise e return status def available(self): @@ -209,10 +210,7 @@ def initGurobi(self): self.model = gp.Model(env=self.env) # Environment handled by user or default Env else: - if self.env is not None: - self.model = gp.Model(env=self.env) - else: - self.model = gp.Model() + self.model = gp.Model(env=self.env) # Set solver parameters for param, value in self.solver_params.items(): self.model.setParam(param, value) @@ -277,16 +275,19 @@ def buildSolverModel(self, lp): list(constraint.values()), [v.solverVar for v in constraint.keys()] ) if constraint.sense == constants.LpConstraintLE: - relation = gp.GRB.LESS_EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr <= -constraint.constant, name=name + ) elif constraint.sense == constants.LpConstraintGE: - relation = gp.GRB.GREATER_EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr >= -constraint.constant, name=name + ) elif constraint.sense == constants.LpConstraintEQ: - relation = gp.GRB.EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr == -constraint.constant, name=name + ) else: raise PulpSolverError("Detected an invalid constraint type") - constraint.solverConstraint = lp.solverModel.addConstr( - expr, relation, -constraint.constant, name - ) lp.solverModel.update() def actualSolve(self, lp, callback=None): diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index f9e2d940..b3ad7845 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -1211,7 +1211,12 @@ def test_measuring_solving_time(self): time_limit = 10 solver_settings = dict( - PULP_CBC_CMD=30, COIN_CMD=30, SCIP_CMD=30, GUROBI_CMD=50, CPLEX_CMD=50 + PULP_CBC_CMD=30, + COIN_CMD=30, + SCIP_CMD=30, + GUROBI_CMD=50, + CPLEX_CMD=50, + GUROBI=50, ) bins = solver_settings.get(self.solver.name) if bins is None: @@ -1233,6 +1238,9 @@ def test_measuring_solving_time(self): delta=delta, msg="optimization time for solver {}".format(self.solver.name), ) + self.assertTrue(prob.objective.value() is not None) + for v in prob.variables(): + self.assertTrue(v.varValue is not None) def test_invalid_var_names(self): prob = LpProblem(self._testMethodName, const.LpMinimize) From b95e06143c7d20ac817bfc21bf31730ed1cb450d Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Tue, 2 May 2023 11:47:23 +0100 Subject: [PATCH 06/11] Address reviewer, rename tests, fix broken tests --- pulp/apis/gurobi_api.py | 52 ++++++++++++++++----------------- pulp/tests/test_gurobipy_env.py | 12 ++++---- pulp/tests/test_pulp.py | 12 ++++++-- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 8d1480da..f09645f9 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -158,35 +158,30 @@ def findSolutionValues(self, lp): var.isModified = False status = gurobiLpStatus.get(solutionStatus, constants.LpStatusUndefined) lp.assignStatus(status) - if status in [constants.LpStatusOptimal, constants.LpStatusNotSolved]: - try: - # populate pulp solution values + if model.SolCount >= 1: + # populate pulp solution values + for var, value in zip( + lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) + ): + var.varValue = value + # populate pulp constraints slack + for constr, value in zip( + lp.constraints.values(), + model.getAttr(GRB.Attr.Slack, model.getConstrs()), + ): + constr.slack = value + # put pi and slack variables against the constraints + if not model.IsMIP: for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) + lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) ): - var.varValue = value + var.dj = value - # populate pulp constraints slack for constr, value in zip( lp.constraints.values(), - model.getAttr(GRB.Attr.Slack, model.getConstrs()), + model.getAttr(GRB.Attr.Pi, model.getConstrs()), ): - constr.slack = value - - if not model.getAttr(GRB.Attr.IsMIP): - for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) - ): - var.dj = value - - # put pi and slack variables against the constraints - for constr, value in zip( - lp.constraints.values(), - model.getAttr(GRB.Attr.Pi, model.getConstrs()), - ): - constr.pi = value - except gp.GurobiError as e: - raise e + constr.pi = value return status def available(self): @@ -214,7 +209,6 @@ def initGurobi(self): # Set solver parameters for param, value in self.solver_params.items(): self.model.setParam(param, value) - # for param, value in self.env_options: except gp.GurobiError as e: raise e @@ -246,6 +240,8 @@ def buildSolverModel(self, lp): lp.solverModel.setParam("LogFile", logPath) log.debug("add the variables to the problem") + lp.solverModel.update() + nvars = lp.solverModel.NumVars for var in lp.variables(): lowBound = var.lowBound if lowBound is None: @@ -257,9 +253,11 @@ def buildSolverModel(self, lp): varType = gp.GRB.CONTINUOUS if var.cat == constants.LpInteger and self.mip: varType = gp.GRB.INTEGER - var.solverVar = lp.solverModel.addVar( - lowBound, upBound, vtype=varType, obj=obj, name=var.name - ) + # only add variable once, ow new variable will be created. + if not hasattr(var, "solverVar") or nvars == 0: + var.solverVar = lp.solverModel.addVar( + lowBound, upBound, vtype=varType, obj=obj, name=var.name + ) if self.optionsDict.get("warmStart", False): # Once lp.variables() has been used at least once in the building of the model. # we can use the lp._variables with the cache. diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index 9e3b6a29..e51240be 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -26,7 +26,7 @@ def setUp(self): self.options = {"OutputFlag": 1} self.env_options = {"MemLimit": 1} - def testGpEnv(self): + def test_gp_env(self): # Using gp.Env within a context manager with gp.Env(params=self.env_options) as env: prob = generate_lp() @@ -35,7 +35,7 @@ def testGpEnv(self): solver.close() check_dummy_env() - def testMultipleGpEnv(self): + def test_multiple_gp_env(self): # Using the same env multiple times with gp.Env() as env: solver = GUROBI(msg=True, env=env) @@ -51,7 +51,7 @@ def testMultipleGpEnv(self): check_dummy_env() @unittest.SkipTest - def testBackwardCompatibility(self): + def test_backward_compatibility(self): """ Backward compatibility check as previously the environment was not being freed. On a single-use license this passes (fails to initialise a dummy @@ -65,7 +65,7 @@ def testBackwardCompatibility(self): gp.disposeDefaultEnv() solver.close() - def testManageEnvTrue(self): + def test_manage_env(self): solver = GUROBI(msg=True, manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) @@ -73,7 +73,7 @@ def testManageEnvTrue(self): solver.close() check_dummy_env() - def testMultipleSolves(self): + def test_multiple_solves(self): solver = GUROBI(msg=True, manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) @@ -88,7 +88,7 @@ def testMultipleSolves(self): check_dummy_env() @unittest.SkipTest - def testLeak(self): + def test_leak(self): """ Check that we cannot initialise environments after a memory leak. On a single-use license this passes (fails to initialise a dummy env with a diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index b3ad7845..bbdb1773 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -1254,9 +1254,15 @@ def test_invalid_var_names(self): prob += -y + z == 7, "c3" prob += w >= 0, "c4" print("\t Testing invalid var names") - pulpTestCheck( - prob, self.solver, [const.LpStatusOptimal], {x: 4, y: -1, z: 6, w: 0} - ) + if self.solver.name not in [ + "GUROBI_CMD", # end is a key-word for LP files + ]: + pulpTestCheck( + prob, + self.solver, + [const.LpStatusOptimal], + {x: 4, y: -1, z: 6, w: 0}, + ) def test_LpVariable_indexs_param(self): """ From 3d00c59cad1b600a5152064d2e656f7db6733f95 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Wed, 3 May 2023 15:45:32 +0100 Subject: [PATCH 07/11] Update docs, add test --- pulp/apis/gurobi_api.py | 63 ++++++++++++++++++++++++++++++--- pulp/tests/test_gurobipy_env.py | 10 +++++- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 67015eaa..a2d550c9 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -87,10 +87,58 @@ def __init__( :param bool warmStart: if True, the solver will use the current value of variables as a start :param str logPath: path to the log file :param float epgap: deprecated for gapRel - :param gp.Env env: gurobipyEnv. Default None. - :param dict envOptions: environment options - :param bool manageEnv: if False, assume gp.Env's is handled - internally and outside of context manager + :param gp.Env env: Gurobi environment to use. Default None. + :param dict envOptions: environment options. + :param bool manageEnv: if False, assume the environment is handled by the user. + + + If ``manageEnv`` is set to True, the ``GUROBI`` object creates a + local Gurobi environment and manages all associated Gurobi + resources. Importantly, this enables Gurobi licenses to be freed + and connections terminated when the ``.close()`` function is called + (this function always disposes of the Gurobi model, and the + environment):: + + solver = GUROBI(manageEnv=True) + prob.solve(solver) + solver.close() # Must be called to free Gurobi resources. + # All Gurobi models and environments are freed + + ``manageEnv=True`` is required when setting license or connection + parameters. The ``envOptions`` argument is used to pass parameters + to the Gurobi environment. For example, to connect to a Gurobi + Cluster Manager:: + + options = { + "CSManager": "", + "CSAPIAccessID": "", + "CSAPISecret": "", + } + solver = GUROBI(manageEnv=True, envOptions=options) + solver.close() + # Compute server connection terminated + + Alternatively, one can also pass a ``gp.Env`` object. In this case, + to be safe, one should still call ``.close()`` to dispose of the + model:: + + with gp.Env(params=options) as env: + # Pass environment as a parameter + solver = GUROBI(env=env) + prob.solve(solver) + solver.close() + # Still call `close` as this disposes the model which is required to correctly free env + + If ``manageEnv`` is set to False (the default), the ``GUROBI`` + object uses the global default Gurobi environment which will be + freed once the object is deleted. In this case, one can still call + ``.close()`` to dispose of the model:: + + solver = GUROBI() + prob.solve(solver) + # The global default environment and model remain active + solver.close() + # Only the global default environment remains active """ self.env = env self.env_options = envOptions if envOptions else {} @@ -128,11 +176,16 @@ def __del__(self): self.close() def close(self): + """ + Must be called when internal Gurobi model and/or environment + requires disposing. The environment (default or otherwise) will be + disposed only if ``manageEnv`` is set to True. + """ if not self.init_gurobi: return + self.model.dispose() if self.manage_env: self.env.dispose() - self.model.dispose() def findSolutionValues(self, lp): model = lp.solverModel diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index e51240be..ee65e71a 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -2,7 +2,6 @@ import gurobipy as gp - from pulp import GUROBI, GUROBI_CMD, LpProblem, LpVariable, const @@ -35,6 +34,15 @@ def test_gp_env(self): solver.close() check_dummy_env() + @unittest.SkipTest + def test_gp_env_no_close(self): + # Not closing results in an error for a single use license. + with gp.Env(params=self.env_options) as env: + prob = generate_lp() + solver = GUROBI(msg=True, env=env, **self.options) + prob.solve(solver) + self.assertRaises(gp.GurobiError, check_dummy_env) + def test_multiple_gp_env(self): # Using the same env multiple times with gp.Env() as env: From 73786c08dc782d424756b7d6951a6d4a19c8c240 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Tue, 1 Aug 2023 09:06:31 +0100 Subject: [PATCH 08/11] Remove unused import --- pulp/tests/test_gurobipy_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index ee65e71a..5ba4351e 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -2,7 +2,7 @@ import gurobipy as gp -from pulp import GUROBI, GUROBI_CMD, LpProblem, LpVariable, const +from pulp import GUROBI, LpProblem, LpVariable, const def check_dummy_env(): From b15e899fd266511022b2c26c7db24e2e11aeff41 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:27:18 +0100 Subject: [PATCH 09/11] Add gurobipy installation to CI --- .github/workflows/pythonpackage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a96d8037..2921729c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,6 +26,9 @@ jobs: run: | python -m pip install --upgrade pip pip install . + - name: Install gurobipy + run: | + python -m pip install gurobipy - name: Install highspy if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest' run: | From edba8c24d29def245da39061f09d294e965d4588 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Fri, 4 Aug 2023 12:56:04 +0100 Subject: [PATCH 10/11] Test changes to work with the pip-license. - Add decorator to skip `test_measuring_solving_time` as it exceeds the pip-license size limitations. - Add `test_gurobipy_env` to pulptest which will be skipped if `gurobipy` is not installed. --- pulp/tests/run_tests.py | 5 ++++- pulp/tests/test_gurobipy_env.py | 32 +++++++++++++++++++------------- pulp/tests/test_pulp.py | 26 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pulp/tests/run_tests.py b/pulp/tests/run_tests.py index 87e0d892..c6dd7647 100644 --- a/pulp/tests/run_tests.py +++ b/pulp/tests/run_tests.py @@ -1,6 +1,6 @@ import unittest import pulp -from pulp.tests import test_pulp, test_examples +from pulp.tests import test_pulp, test_examples, test_gurobipy_env def pulpTestAll(test_docs=False): @@ -19,6 +19,9 @@ def get_test_suite(test_docs=False): # we get suite with all PuLP tests pulp_solver_tests = loader.loadTestsFromModule(test_pulp) suite_all.addTests(pulp_solver_tests) + # Add tests for gurobipy env + gurobipy_env = loader.loadTestsFromModule(test_gurobipy_env) + suite_all.addTests(gurobipy_env) # We add examples and docs tests if test_docs: docs_examples = loader.loadTestsFromTestCase(test_examples.Examples_DocsTests) diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index 5ba4351e..52a50375 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -1,9 +1,13 @@ import unittest -import gurobipy as gp - from pulp import GUROBI, LpProblem, LpVariable, const +try: + import gurobipy as gp + from gurobipy import GRB +except ImportError: + gp = None + def check_dummy_env(): with gp.Env(): @@ -22,14 +26,16 @@ def generate_lp() -> LpProblem: class GurobiEnvTests(unittest.TestCase): def setUp(self): - self.options = {"OutputFlag": 1} + if gp is None: + self.skipTest("Skipping all tests in test_gurobipy_env.py") + self.options = {"OutputFlag": 0} self.env_options = {"MemLimit": 1} def test_gp_env(self): # Using gp.Env within a context manager with gp.Env(params=self.env_options) as env: prob = generate_lp() - solver = GUROBI(msg=True, env=env, **self.options) + solver = GUROBI(env=env, **self.options) prob.solve(solver) solver.close() check_dummy_env() @@ -39,19 +45,19 @@ def test_gp_env_no_close(self): # Not closing results in an error for a single use license. with gp.Env(params=self.env_options) as env: prob = generate_lp() - solver = GUROBI(msg=True, env=env, **self.options) + solver = GUROBI(env=env, **self.options) prob.solve(solver) self.assertRaises(gp.GurobiError, check_dummy_env) def test_multiple_gp_env(self): # Using the same env multiple times with gp.Env() as env: - solver = GUROBI(msg=True, env=env) + solver = GUROBI(env=env) prob = generate_lp() prob.solve(solver) solver.close() - solver2 = GUROBI(msg=True, env=env) + solver2 = GUROBI(env=env) prob2 = generate_lp() prob2.solve(solver2) solver2.close() @@ -65,7 +71,7 @@ def test_backward_compatibility(self): freed. On a single-use license this passes (fails to initialise a dummy env). """ - solver = GUROBI(msg=True, **self.options) + solver = GUROBI(**self.options) prob = generate_lp() prob.solve(solver) @@ -74,7 +80,7 @@ def test_backward_compatibility(self): solver.close() def test_manage_env(self): - solver = GUROBI(msg=True, manageEnv=True, **self.options) + solver = GUROBI(manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) @@ -82,14 +88,14 @@ def test_manage_env(self): check_dummy_env() def test_multiple_solves(self): - solver = GUROBI(msg=True, manageEnv=True, **self.options) + solver = GUROBI(manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) solver.close() check_dummy_env() - solver2 = GUROBI(msg=True, manageEnv=True, **self.options) + solver2 = GUROBI(manageEnv=True, **self.options) prob.solve(solver2) solver2.close() @@ -102,14 +108,14 @@ def test_leak(self): single-use license this passes (fails to initialise a dummy env with a memory leak). """ - solver = GUROBI(msg=True, **self.options) + solver = GUROBI(**self.options) prob = generate_lp() prob.solve(solver) tmp = solver.model solver.close() - solver2 = GUROBI(msg=True, **self.options) + solver2 = GUROBI(**self.options) prob2 = generate_lp() prob2.solve(solver2) diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index e88f2422..c18912cf 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -10,8 +10,14 @@ from pulp import constants as const from pulp.tests.bin_packing_problem import create_bin_packing_problem from pulp.utilities import makeDict +import functools import unittest +try: + import gurobipy as gp +except ImportError: + gp = None + # from: http://lpsolve.sourceforge.net/5.5/mps-format.htm EXAMPLE_MPS_RHS56 = """NAME TESTPROB ROWS @@ -37,6 +43,25 @@ """ +def gurobi_test(test_item): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + if gp is None: + raise unittest.SkipTest("No gurobipy, can't check license") + try: + test_item(*args, **kwargs) + except gp.GurobiError as ge: + # Skip the test if the failure was due to licensing + if ge.errno == gp.GRB.Error.SIZE_LIMIT_EXCEEDED: + raise unittest.SkipTest("Size-limited Gurobi license") + if ge.errno == gp.GRB.Error.NO_LICENSE: + raise unittest.SkipTest("No Gurobi license") + # Otherwise, let the error go through as-is + raise + + return skip_wrapper + + def dumpTestProblem(prob): try: prob.writeLP("debug.lp") @@ -1207,6 +1232,7 @@ def add_const(prob): self.assertRaises(TypeError, add_const, prob=prob) + @gurobi_test def test_measuring_solving_time(self): print("\t Testing measuring optimization time") From 78b54d335347fab0767ec5a3fd373ac4f2a078a2 Mon Sep 17 00:00:00 2001 From: torressa <23246013+torressa@users.noreply.github.com> Date: Mon, 7 Aug 2023 18:44:34 +0100 Subject: [PATCH 11/11] Remove output from tests --- pulp/apis/gurobi_api.py | 3 ++- pulp/tests/test_gurobipy_env.py | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index a2d550c9..05332da2 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -170,6 +170,7 @@ def __init__( if self.manage_env: self.env_options["OutputFlag"] = 0 else: + self.env_options["OutputFlag"] = 0 self.solver_params["OutputFlag"] = 0 def __del__(self): @@ -240,7 +241,7 @@ def findSolutionValues(self, lp): def available(self): """True if the solver is available""" try: - with gp.Env(): + with gp.Env(params=self.env_options): pass except gurobipy.GurobiError as e: warnings.warn(f"GUROBI error: {e}.") diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py index 52a50375..9246f387 100644 --- a/pulp/tests/test_gurobipy_env.py +++ b/pulp/tests/test_gurobipy_env.py @@ -10,7 +10,7 @@ def check_dummy_env(): - with gp.Env(): + with gp.Env(params={"OutputFlag": 0}): pass @@ -28,14 +28,14 @@ class GurobiEnvTests(unittest.TestCase): def setUp(self): if gp is None: self.skipTest("Skipping all tests in test_gurobipy_env.py") - self.options = {"OutputFlag": 0} - self.env_options = {"MemLimit": 1} + self.options = {"Method": 0} + self.env_options = {"MemLimit": 1, "OutputFlag": 0} def test_gp_env(self): # Using gp.Env within a context manager with gp.Env(params=self.env_options) as env: prob = generate_lp() - solver = GUROBI(env=env, **self.options) + solver = GUROBI(msg=False, env=env, **self.options) prob.solve(solver) solver.close() check_dummy_env() @@ -45,19 +45,19 @@ def test_gp_env_no_close(self): # Not closing results in an error for a single use license. with gp.Env(params=self.env_options) as env: prob = generate_lp() - solver = GUROBI(env=env, **self.options) + solver = GUROBI(msg=False, env=env, **self.options) prob.solve(solver) self.assertRaises(gp.GurobiError, check_dummy_env) def test_multiple_gp_env(self): # Using the same env multiple times - with gp.Env() as env: - solver = GUROBI(env=env) + with gp.Env(params=self.env_options) as env: + solver = GUROBI(msg=False, env=env) prob = generate_lp() prob.solve(solver) solver.close() - solver2 = GUROBI(env=env) + solver2 = GUROBI(msg=False, env=env) prob2 = generate_lp() prob2.solve(solver2) solver2.close() @@ -71,7 +71,7 @@ def test_backward_compatibility(self): freed. On a single-use license this passes (fails to initialise a dummy env). """ - solver = GUROBI(**self.options) + solver = GUROBI(msg=False, **self.options) prob = generate_lp() prob.solve(solver) @@ -80,7 +80,7 @@ def test_backward_compatibility(self): solver.close() def test_manage_env(self): - solver = GUROBI(manageEnv=True, **self.options) + solver = GUROBI(msg=False, manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) @@ -88,14 +88,14 @@ def test_manage_env(self): check_dummy_env() def test_multiple_solves(self): - solver = GUROBI(manageEnv=True, **self.options) + solver = GUROBI(msg=False, manageEnv=True, **self.options) prob = generate_lp() prob.solve(solver) solver.close() check_dummy_env() - solver2 = GUROBI(manageEnv=True, **self.options) + solver2 = GUROBI(msg=False, manageEnv=True, **self.options) prob.solve(solver2) solver2.close() @@ -108,14 +108,14 @@ def test_leak(self): single-use license this passes (fails to initialise a dummy env with a memory leak). """ - solver = GUROBI(**self.options) + solver = GUROBI(msg=False, **self.options) prob = generate_lp() prob.solve(solver) tmp = solver.model solver.close() - solver2 = GUROBI(**self.options) + solver2 = GUROBI(msg=False, **self.options) prob2 = generate_lp() prob2.solve(solver2)