diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 51bc5aca..14c70996 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,6 +26,9 @@ jobs: run: | python -m pip install --upgrade pip pip install . + - name: Install gurobipy + run: | + python -m pip install gurobipy # - name: Install highspy # if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest' # run: | diff --git a/pulp/apis/__init__.py b/pulp/apis/__init__.py index 1ece3ad0..ec195c2f 100644 --- a/pulp/apis/__init__.py +++ b/pulp/apis/__init__.py @@ -145,10 +145,13 @@ def listSolvers(onlyAvailable=False): :return: list of solver names :rtype: list """ - solvers = [s() for s in _all_solvers] - if onlyAvailable: - return [solver.name for solver in solvers if solver.available()] - return [solver.name for solver in solvers] + result = [] + for s in _all_solvers: + solver = s() + if (not onlyAvailable) or solver.available(): + result.append(solver.name) + del solver + return result # DEPRECATED aliases: diff --git a/pulp/apis/gurobi_api.py b/pulp/apis/gurobi_api.py index 38299df8..05332da2 100644 --- a/pulp/apis/gurobi_api.py +++ b/pulp/apis/gurobi_api.py @@ -33,7 +33,7 @@ import warnings # to import the gurobipy name into the module scope -gurobipy = None +gp = None class GUROBI(LpSolver): @@ -46,12 +46,13 @@ class GUROBI(LpSolver): """ name = "GUROBI" + env = None try: sys.path.append(gurobi_path) # to import the name into the module scope - global gurobipy - import gurobipy + global gp + import gurobipy as gp except: # FIXME: Bug because gurobi returns # a gurobi exception on failed imports def available(self): @@ -73,6 +74,9 @@ def __init__( gapRel=None, warmStart=False, logPath=None, + env=None, + envOptions=None, + manageEnv=False, **solverParams, ): """ @@ -83,13 +87,74 @@ def __init__( :param bool warmStart: if True, the solver will use the current value of variables as a start :param str logPath: path to the log file :param float epgap: deprecated for gapRel + :param gp.Env env: Gurobi environment to use. Default None. + :param dict envOptions: environment options. + :param bool manageEnv: if False, assume the environment is handled by the user. + + + If ``manageEnv`` is set to True, the ``GUROBI`` object creates a + local Gurobi environment and manages all associated Gurobi + resources. Importantly, this enables Gurobi licenses to be freed + and connections terminated when the ``.close()`` function is called + (this function always disposes of the Gurobi model, and the + environment):: + + solver = GUROBI(manageEnv=True) + prob.solve(solver) + solver.close() # Must be called to free Gurobi resources. + # All Gurobi models and environments are freed + + ``manageEnv=True`` is required when setting license or connection + parameters. The ``envOptions`` argument is used to pass parameters + to the Gurobi environment. For example, to connect to a Gurobi + Cluster Manager:: + + options = { + "CSManager": "", + "CSAPIAccessID": "", + "CSAPISecret": "", + } + solver = GUROBI(manageEnv=True, envOptions=options) + solver.close() + # Compute server connection terminated + + Alternatively, one can also pass a ``gp.Env`` object. In this case, + to be safe, one should still call ``.close()`` to dispose of the + model:: + + with gp.Env(params=options) as env: + # Pass environment as a parameter + solver = GUROBI(env=env) + prob.solve(solver) + solver.close() + # Still call `close` as this disposes the model which is required to correctly free env + + If ``manageEnv`` is set to False (the default), the ``GUROBI`` + object uses the global default Gurobi environment which will be + freed once the object is deleted. In this case, one can still call + ``.close()`` to dispose of the model:: + + solver = GUROBI() + prob.solve(solver) + # The global default environment and model remain active + solver.close() + # Only the global default environment remains active """ + self.env = env + self.env_options = envOptions if envOptions else {} + self.manage_env = False if self.env is not None else manageEnv + self.solver_params = solverParams + + self.model = None + self.init_gurobi = False # whether env and model have been initialised + if epgap is not None: warnings.warn("Parameter epgap is being depreciated for gapRel") if gapRel is not None: warnings.warn("Parameter gapRel and epgap passed, using gapRel") else: gapRel = epgap + LpSolver.__init__( self, mip=mip, @@ -99,18 +164,34 @@ def __init__( logPath=logPath, warmStart=warmStart, ) + # set the output of gurobi if not self.msg: - gurobipy.setParam("OutputFlag", 0) + if self.manage_env: + self.env_options["OutputFlag"] = 0 + else: + self.env_options["OutputFlag"] = 0 + self.solver_params["OutputFlag"] = 0 + + def __del__(self): + self.close() - # set the gurobi parameter values - for key, value in solverParams.items(): - gurobipy.setParam(key, value) + def close(self): + """ + Must be called when internal Gurobi model and/or environment + requires disposing. The environment (default or otherwise) will be + disposed only if ``manageEnv`` is set to True. + """ + if not self.init_gurobi: + return + self.model.dispose() + if self.manage_env: + self.env.dispose() def findSolutionValues(self, lp): model = lp.solverModel solutionStatus = model.Status - GRB = gurobipy.GRB + GRB = gp.GRB # TODO: check status for Integer Feasible gurobiLpStatus = { GRB.OPTIMAL: constants.LpStatusOptimal, @@ -131,46 +212,60 @@ def findSolutionValues(self, lp): var.isModified = False status = gurobiLpStatus.get(solutionStatus, constants.LpStatusUndefined) lp.assignStatus(status) - if status != constants.LpStatusOptimal: - return status - - # populate pulp solution values - for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) - ): - var.varValue = value - - # populate pulp constraints slack - for constr, value in zip( - lp.constraints.values(), - model.getAttr(GRB.Attr.Slack, model.getConstrs()), - ): - constr.slack = value - - if not model.getAttr(GRB.Attr.IsMIP): + if model.SolCount >= 1: + # populate pulp solution values for var, value in zip( - lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + lp._variables, model.getAttr(GRB.Attr.X, model.getVars()) ): - var.dj = value - - # put pi and slack variables against the constraints + var.varValue = value + # populate pulp constraints slack for constr, value in zip( lp.constraints.values(), - model.getAttr(GRB.Attr.Pi, model.getConstrs()), + model.getAttr(GRB.Attr.Slack, model.getConstrs()), ): - constr.pi = value - + constr.slack = value + # put pi and slack variables against the constraints + if not model.IsMIP: + for var, value in zip( + lp._variables, model.getAttr(GRB.Attr.RC, model.getVars()) + ): + var.dj = value + + for constr, value in zip( + lp.constraints.values(), + model.getAttr(GRB.Attr.Pi, model.getConstrs()), + ): + constr.pi = value return status def available(self): """True if the solver is available""" try: - gurobipy.setParam("_test", 0) + with gp.Env(params=self.env_options): + pass except gurobipy.GurobiError as e: warnings.warn(f"GUROBI error: {e}.") return False return True + def initGurobi(self): + if self.init_gurobi: + return + else: + self.init_gurobi = True + try: + if self.manage_env: + self.env = gp.Env(params=self.env_options) + self.model = gp.Model(env=self.env) + # Environment handled by user or default Env + else: + self.model = gp.Model(env=self.env) + # Set solver parameters + for param, value in self.solver_params.items(): + self.model.setParam(param, value) + except gp.GurobiError as e: + raise e + def callSolver(self, lp, callback=None): """Solves the problem with gurobi""" # solve the problem @@ -183,7 +278,9 @@ def buildSolverModel(self, lp): Takes the pulp lp model and translates it into a gurobi model """ log.debug("create the gurobi model") - lp.solverModel = gurobipy.Model(lp.name) + self.initGurobi() + self.model.ModelName = lp.name + lp.solverModel = self.model log.debug("set the sense of the problem") if lp.sense == constants.LpMaximize: lp.solverModel.setAttr("ModelSense", -1) @@ -197,20 +294,24 @@ def buildSolverModel(self, lp): lp.solverModel.setParam("LogFile", logPath) log.debug("add the variables to the problem") + lp.solverModel.update() + nvars = lp.solverModel.NumVars for var in lp.variables(): lowBound = var.lowBound if lowBound is None: - lowBound = -gurobipy.GRB.INFINITY + lowBound = -gp.GRB.INFINITY upBound = var.upBound if upBound is None: - upBound = gurobipy.GRB.INFINITY + upBound = gp.GRB.INFINITY obj = lp.objective.get(var, 0.0) - varType = gurobipy.GRB.CONTINUOUS + varType = gp.GRB.CONTINUOUS if var.cat == constants.LpInteger and self.mip: - varType = gurobipy.GRB.INTEGER - var.solverVar = lp.solverModel.addVar( - lowBound, upBound, vtype=varType, obj=obj, name=var.name - ) + varType = gp.GRB.INTEGER + # only add variable once, ow new variable will be created. + if not hasattr(var, "solverVar") or nvars == 0: + var.solverVar = lp.solverModel.addVar( + lowBound, upBound, vtype=varType, obj=obj, name=var.name + ) if self.optionsDict.get("warmStart", False): # Once lp.variables() has been used at least once in the building of the model. # we can use the lp._variables with the cache. @@ -222,20 +323,23 @@ def buildSolverModel(self, lp): log.debug("add the Constraints to the problem") for name, constraint in lp.constraints.items(): # build the expression - expr = gurobipy.LinExpr( + expr = gp.LinExpr( list(constraint.values()), [v.solverVar for v in constraint.keys()] ) if constraint.sense == constants.LpConstraintLE: - relation = gurobipy.GRB.LESS_EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr <= -constraint.constant, name=name + ) elif constraint.sense == constants.LpConstraintGE: - relation = gurobipy.GRB.GREATER_EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr >= -constraint.constant, name=name + ) elif constraint.sense == constants.LpConstraintEQ: - relation = gurobipy.GRB.EQUAL + constraint.solverConstraint = lp.solverModel.addConstr( + expr == -constraint.constant, name=name + ) else: raise PulpSolverError("Detected an invalid constraint type") - constraint.solverConstraint = lp.solverModel.addConstr( - expr, relation, -constraint.constant, name - ) lp.solverModel.update() def actualSolve(self, lp, callback=None): @@ -267,7 +371,7 @@ def actualResolve(self, lp, callback=None): for constraint in lp.constraints.values(): if constraint.modified: constraint.solverConstraint.setAttr( - gurobipy.GRB.Attr.RHS, -constraint.constant + gp.GRB.Attr.RHS, -constraint.constant ) lp.solverModel.update() self.callSolver(lp, callback=callback) diff --git a/pulp/tests/run_tests.py b/pulp/tests/run_tests.py index 87e0d892..c6dd7647 100644 --- a/pulp/tests/run_tests.py +++ b/pulp/tests/run_tests.py @@ -1,6 +1,6 @@ import unittest import pulp -from pulp.tests import test_pulp, test_examples +from pulp.tests import test_pulp, test_examples, test_gurobipy_env def pulpTestAll(test_docs=False): @@ -19,6 +19,9 @@ def get_test_suite(test_docs=False): # we get suite with all PuLP tests pulp_solver_tests = loader.loadTestsFromModule(test_pulp) suite_all.addTests(pulp_solver_tests) + # Add tests for gurobipy env + gurobipy_env = loader.loadTestsFromModule(test_gurobipy_env) + suite_all.addTests(gurobipy_env) # We add examples and docs tests if test_docs: docs_examples = loader.loadTestsFromTestCase(test_examples.Examples_DocsTests) diff --git a/pulp/tests/test_gurobipy_env.py b/pulp/tests/test_gurobipy_env.py new file mode 100644 index 00000000..9246f387 --- /dev/null +++ b/pulp/tests/test_gurobipy_env.py @@ -0,0 +1,122 @@ +import unittest + +from pulp import GUROBI, LpProblem, LpVariable, const + +try: + import gurobipy as gp + from gurobipy import GRB +except ImportError: + gp = None + + +def check_dummy_env(): + with gp.Env(params={"OutputFlag": 0}): + pass + + +def generate_lp() -> LpProblem: + prob = LpProblem("test", const.LpMaximize) + x = LpVariable("x", 0, 1) + y = LpVariable("y", 0, 1) + z = LpVariable("z", 0, 1) + prob += x + y + z, "obj" + prob += x + y + z <= 1, "c1" + return prob + + +class GurobiEnvTests(unittest.TestCase): + def setUp(self): + if gp is None: + self.skipTest("Skipping all tests in test_gurobipy_env.py") + self.options = {"Method": 0} + self.env_options = {"MemLimit": 1, "OutputFlag": 0} + + def test_gp_env(self): + # Using gp.Env within a context manager + with gp.Env(params=self.env_options) as env: + prob = generate_lp() + solver = GUROBI(msg=False, env=env, **self.options) + prob.solve(solver) + solver.close() + check_dummy_env() + + @unittest.SkipTest + def test_gp_env_no_close(self): + # Not closing results in an error for a single use license. + with gp.Env(params=self.env_options) as env: + prob = generate_lp() + solver = GUROBI(msg=False, env=env, **self.options) + prob.solve(solver) + self.assertRaises(gp.GurobiError, check_dummy_env) + + def test_multiple_gp_env(self): + # Using the same env multiple times + with gp.Env(params=self.env_options) as env: + solver = GUROBI(msg=False, env=env) + prob = generate_lp() + prob.solve(solver) + solver.close() + + solver2 = GUROBI(msg=False, env=env) + prob2 = generate_lp() + prob2.solve(solver2) + solver2.close() + + check_dummy_env() + + @unittest.SkipTest + def test_backward_compatibility(self): + """ + Backward compatibility check as previously the environment was not being + freed. On a single-use license this passes (fails to initialise a dummy + env). + """ + solver = GUROBI(msg=False, **self.options) + prob = generate_lp() + prob.solve(solver) + + self.assertRaises(gp.GurobiError, check_dummy_env) + gp.disposeDefaultEnv() + solver.close() + + def test_manage_env(self): + solver = GUROBI(msg=False, manageEnv=True, **self.options) + prob = generate_lp() + prob.solve(solver) + + solver.close() + check_dummy_env() + + def test_multiple_solves(self): + solver = GUROBI(msg=False, manageEnv=True, **self.options) + prob = generate_lp() + prob.solve(solver) + + solver.close() + check_dummy_env() + + solver2 = GUROBI(msg=False, manageEnv=True, **self.options) + prob.solve(solver2) + + solver2.close() + check_dummy_env() + + @unittest.SkipTest + def test_leak(self): + """ + Check that we cannot initialise environments after a memory leak. On a + single-use license this passes (fails to initialise a dummy env with a + memory leak). + """ + solver = GUROBI(msg=False, **self.options) + prob = generate_lp() + prob.solve(solver) + + tmp = solver.model + solver.close() + + solver2 = GUROBI(msg=False, **self.options) + + prob2 = generate_lp() + prob2.solve(solver2) + self.assertRaises(gp.GurobiError, check_dummy_env) diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index 0ce442d7..c18912cf 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -10,8 +10,14 @@ from pulp import constants as const from pulp.tests.bin_packing_problem import create_bin_packing_problem from pulp.utilities import makeDict +import functools import unittest +try: + import gurobipy as gp +except ImportError: + gp = None + # from: http://lpsolve.sourceforge.net/5.5/mps-format.htm EXAMPLE_MPS_RHS56 = """NAME TESTPROB ROWS @@ -37,6 +43,25 @@ """ +def gurobi_test(test_item): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + if gp is None: + raise unittest.SkipTest("No gurobipy, can't check license") + try: + test_item(*args, **kwargs) + except gp.GurobiError as ge: + # Skip the test if the failure was due to licensing + if ge.errno == gp.GRB.Error.SIZE_LIMIT_EXCEEDED: + raise unittest.SkipTest("Size-limited Gurobi license") + if ge.errno == gp.GRB.Error.NO_LICENSE: + raise unittest.SkipTest("No Gurobi license") + # Otherwise, let the error go through as-is + raise + + return skip_wrapper + + def dumpTestProblem(prob): try: prob.writeLP("debug.lp") @@ -1207,6 +1232,7 @@ def add_const(prob): self.assertRaises(TypeError, add_const, prob=prob) + @gurobi_test def test_measuring_solving_time(self): print("\t Testing measuring optimization time") @@ -1217,6 +1243,7 @@ def test_measuring_solving_time(self): SCIP_CMD=30, GUROBI_CMD=50, CPLEX_CMD=50, + GUROBI=50, HiGHS=50, ) bins = solver_settings.get(self.solver.name) @@ -1237,6 +1264,9 @@ def test_measuring_solving_time(self): delta=delta, msg=f"optimization time for solver {self.solver.name}", ) + self.assertTrue(prob.objective.value() is not None) + for v in prob.variables(): + self.assertTrue(v.varValue is not None) def test_invalid_var_names(self): prob = LpProblem(self._testMethodName, const.LpMinimize) @@ -1250,9 +1280,15 @@ def test_invalid_var_names(self): prob += -y + z == 7, "c3" prob += w >= 0, "c4" print("\t Testing invalid var names") - pulpTestCheck( - prob, self.solver, [const.LpStatusOptimal], {x: 4, y: -1, z: 6, w: 0} - ) + if self.solver.name not in [ + "GUROBI_CMD", # end is a key-word for LP files + ]: + pulpTestCheck( + prob, + self.solver, + [const.LpStatusOptimal], + {x: 4, y: -1, z: 6, w: 0}, + ) def test_LpVariable_indexs_param(self): """