Gurobipy Environment Handling (#588)

* Add environment handling and context manager

* Dispose of solvers in listSolvers

* Close environment in Gurobi availability check

* Add environment handling and context manager

* [coin-or/pulp/586] Fix and add test. Remove deprecation message for model.addConstr

* Address reviewer, rename tests, fix broken tests

* Update docs, add test

* Remove unused import

* Add gurobipy installation to CI

* Test changes to work with the pip-license.

- Add decorator to skip `test_measuring_solving_time` as it exceeds the
  pip-license size limitations.
- Add `test_gurobipy_env` to pulptest which will be skipped if
  `gurobipy` is not installed.

* Remove output from tests

---------

Co-authored-by: Simon Bowly <bowly@gurobi.com>

.github/workflows/pythonpackage.yml

@@ -26,6 +26,9 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .
+      - name: Install gurobipy
+        run: |
+          python -m pip install gurobipy
 #      - name: Install highspy
 #        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
 #        run: |

pulp/apis/__init__.py

@@ -145,10 +145,13 @@ def listSolvers(onlyAvailable=False):
     :return: list of solver names
     :rtype: list
     """
-    solvers = [s() for s in _all_solvers]
-    if onlyAvailable:
-        return [solver.name for solver in solvers if solver.available()]
-    return [solver.name for solver in solvers]
+    result = []
+    for s in _all_solvers:
+        solver = s()
+        if (not onlyAvailable) or solver.available():
+            result.append(solver.name)
+        del solver
+    return result
 
 
 # DEPRECATED aliases:

pulp/apis/gurobi_api.py

@@ -33,7 +33,7 @@
 import warnings
 
 # to import the gurobipy name into the module scope
-gurobipy = None
+gp = None
 
 
 class GUROBI(LpSolver):
@@ -46,12 +46,13 @@ class GUROBI(LpSolver):
     """
 
     name = "GUROBI"
+    env = None
 
     try:
         sys.path.append(gurobi_path)
         # to import the name into the module scope
-        global gurobipy
-        import gurobipy
+        global gp
+        import gurobipy as gp
     except:  # FIXME: Bug because gurobi returns
         #  a gurobi exception on failed imports
         def available(self):
@@ -73,6 +74,9 @@ def __init__(
             gapRel=None,
             warmStart=False,
             logPath=None,
+            env=None,
+            envOptions=None,
+            manageEnv=False,
             **solverParams,
         ):
             """
@@ -83,13 +87,74 @@ def __init__(
             :param bool warmStart: if True, the solver will use the current value of variables as a start
             :param str logPath: path to the log file
             :param float epgap: deprecated for gapRel
+            :param gp.Env env: Gurobi environment to use. Default None.
+            :param dict envOptions: environment options.
+            :param bool manageEnv: if False, assume the environment is handled by the user.
+
+
+            If ``manageEnv`` is set to True, the ``GUROBI`` object creates a
+            local Gurobi environment and manages all associated Gurobi
+            resources. Importantly, this enables Gurobi licenses to be freed
+            and connections terminated when the ``.close()`` function is called
+            (this function always disposes of the Gurobi model, and the
+            environment)::
+
+                solver = GUROBI(manageEnv=True)
+                prob.solve(solver)
+                solver.close() # Must be called to free Gurobi resources.
+                # All Gurobi models and environments are freed
+
+            ``manageEnv=True`` is required when setting license or connection
+            parameters. The ``envOptions`` argument is used to pass parameters
+            to the Gurobi environment. For example, to connect to a Gurobi
+            Cluster Manager::
+
+                options = {
+                    "CSManager": "<url>",
+                    "CSAPIAccessID": "<access-id>",
+                    "CSAPISecret": "<api-key>",
+                }
+                solver = GUROBI(manageEnv=True, envOptions=options)
+                solver.close()
+                # Compute server connection terminated
+
+            Alternatively, one can also pass a ``gp.Env`` object. In this case,
+            to be safe, one should still call ``.close()`` to dispose of the
+            model::
+
+                with gp.Env(params=options) as env:
+                    # Pass environment as a parameter
+                    solver = GUROBI(env=env)
+                    prob.solve(solver)
+                    solver.close()
+                    # Still call `close` as this disposes the model which is required to correctly free env
+
+            If ``manageEnv`` is set to False (the default), the ``GUROBI``
+            object uses the global default Gurobi environment which will be
+            freed once the object is deleted. In this case, one can still call
+            ``.close()`` to dispose of the model::
+
+                solver = GUROBI()
+                prob.solve(solver)
+                # The global default environment and model remain active
+                solver.close()
+                # Only the global default environment remains active
             """
+            self.env = env
+            self.env_options = envOptions if envOptions else {}
+            self.manage_env = False if self.env is not None else manageEnv
+            self.solver_params = solverParams
+
+            self.model = None
+            self.init_gurobi = False  # whether env and model have been initialised
+
             if epgap is not None:
                 warnings.warn("Parameter epgap is being depreciated for gapRel")
                 if gapRel is not None:
                     warnings.warn("Parameter gapRel and epgap passed, using gapRel")
                 else:
                     gapRel = epgap
+
             LpSolver.__init__(
                 self,
                 mip=mip,
@@ -99,18 +164,34 @@ def __init__(
                 logPath=logPath,
                 warmStart=warmStart,
             )
+
             # set the output of gurobi
             if not self.msg:
-                gurobipy.setParam("OutputFlag", 0)
+                if self.manage_env:
+                    self.env_options["OutputFlag"] = 0
+                else:
+                    self.env_options["OutputFlag"] = 0
+                    self.solver_params["OutputFlag"] = 0
+
+        def __del__(self):
+            self.close()
 
-            # set the gurobi parameter values
-            for key, value in solverParams.items():
-                gurobipy.setParam(key, value)
+        def close(self):
+            """
+            Must be called when internal Gurobi model and/or environment
+            requires disposing. The environment (default or otherwise) will be
+            disposed only if ``manageEnv`` is set to True.
+            """
+            if not self.init_gurobi:
+                return
+            self.model.dispose()
+            if self.manage_env:
+                self.env.dispose()
 
         def findSolutionValues(self, lp):
             model = lp.solverModel
             solutionStatus = model.Status
-            GRB = gurobipy.GRB
+            GRB = gp.GRB
             # TODO: check status for Integer Feasible
             gurobiLpStatus = {
                 GRB.OPTIMAL: constants.LpStatusOptimal,
@@ -131,46 +212,60 @@ def findSolutionValues(self, lp):
                 var.isModified = False
             status = gurobiLpStatus.get(solutionStatus, constants.LpStatusUndefined)
             lp.assignStatus(status)
-            if status != constants.LpStatusOptimal:
-                return status
-
-            # populate pulp solution values
-            for var, value in zip(
-                lp._variables, model.getAttr(GRB.Attr.X, model.getVars())
-            ):
-                var.varValue = value
-
-            # populate pulp constraints slack
-            for constr, value in zip(
-                lp.constraints.values(),
-                model.getAttr(GRB.Attr.Slack, model.getConstrs()),
-            ):
-                constr.slack = value
-
-            if not model.getAttr(GRB.Attr.IsMIP):
+            if model.SolCount >= 1:
+                # populate pulp solution values
                 for var, value in zip(
-                    lp._variables, model.getAttr(GRB.Attr.RC, model.getVars())
+                    lp._variables, model.getAttr(GRB.Attr.X, model.getVars())
                 ):
-                    var.dj = value
-
-                # put pi and slack variables against the constraints
+                    var.varValue = value
+                # populate pulp constraints slack
                 for constr, value in zip(
                     lp.constraints.values(),
-                    model.getAttr(GRB.Attr.Pi, model.getConstrs()),
+                    model.getAttr(GRB.Attr.Slack, model.getConstrs()),
                 ):
-                    constr.pi = value
-
+                    constr.slack = value
+                # put pi and slack variables against the constraints
+                if not model.IsMIP:
+                    for var, value in zip(
+                        lp._variables, model.getAttr(GRB.Attr.RC, model.getVars())
+                    ):
+                        var.dj = value
+
+                    for constr, value in zip(
+                        lp.constraints.values(),
+                        model.getAttr(GRB.Attr.Pi, model.getConstrs()),
+                    ):
+                        constr.pi = value
             return status
 
         def available(self):
             """True if the solver is available"""
             try:
-                gurobipy.setParam("_test", 0)
+                with gp.Env(params=self.env_options):
+                    pass
             except gurobipy.GurobiError as e:
                 warnings.warn(f"GUROBI error: {e}.")
                 return False
             return True
 
+        def initGurobi(self):
+            if self.init_gurobi:
+                return
+            else:
+                self.init_gurobi = True
+            try:
+                if self.manage_env:
+                    self.env = gp.Env(params=self.env_options)
+                    self.model = gp.Model(env=self.env)
+                # Environment handled by user or default Env
+                else:
+                    self.model = gp.Model(env=self.env)
+                # Set solver parameters
+                for param, value in self.solver_params.items():
+                    self.model.setParam(param, value)
+            except gp.GurobiError as e:
+                raise e
+
         def callSolver(self, lp, callback=None):
             """Solves the problem with gurobi"""
             # solve the problem
@@ -183,7 +278,9 @@ def buildSolverModel(self, lp):
             Takes the pulp lp model and translates it into a gurobi model
             """
             log.debug("create the gurobi model")
-            lp.solverModel = gurobipy.Model(lp.name)
+            self.initGurobi()
+            self.model.ModelName = lp.name
+            lp.solverModel = self.model
             log.debug("set the sense of the problem")
             if lp.sense == constants.LpMaximize:
                 lp.solverModel.setAttr("ModelSense", -1)
@@ -197,20 +294,24 @@ def buildSolverModel(self, lp):
                 lp.solverModel.setParam("LogFile", logPath)
 
             log.debug("add the variables to the problem")
+            lp.solverModel.update()
+            nvars = lp.solverModel.NumVars
             for var in lp.variables():
                 lowBound = var.lowBound
                 if lowBound is None:
-                    lowBound = -gurobipy.GRB.INFINITY
+                    lowBound = -gp.GRB.INFINITY
                 upBound = var.upBound
                 if upBound is None:
-                    upBound = gurobipy.GRB.INFINITY
+                    upBound = gp.GRB.INFINITY
                 obj = lp.objective.get(var, 0.0)
-                varType = gurobipy.GRB.CONTINUOUS
+                varType = gp.GRB.CONTINUOUS
                 if var.cat == constants.LpInteger and self.mip:
-                    varType = gurobipy.GRB.INTEGER
-                var.solverVar = lp.solverModel.addVar(
-                    lowBound, upBound, vtype=varType, obj=obj, name=var.name
-                )
+                    varType = gp.GRB.INTEGER
+                # only add variable once, ow new variable will be created.
+                if not hasattr(var, "solverVar") or nvars == 0:
+                    var.solverVar = lp.solverModel.addVar(
+                        lowBound, upBound, vtype=varType, obj=obj, name=var.name
+                    )
             if self.optionsDict.get("warmStart", False):
                 # Once lp.variables() has been used at least once in the building of the model.
                 # we can use the lp._variables with the cache.
@@ -222,20 +323,23 @@ def buildSolverModel(self, lp):
             log.debug("add the Constraints to the problem")
             for name, constraint in lp.constraints.items():
                 # build the expression
-                expr = gurobipy.LinExpr(
+                expr = gp.LinExpr(
                     list(constraint.values()), [v.solverVar for v in constraint.keys()]
                 )
                 if constraint.sense == constants.LpConstraintLE:
-                    relation = gurobipy.GRB.LESS_EQUAL
+                    constraint.solverConstraint = lp.solverModel.addConstr(
+                        expr <= -constraint.constant, name=name
+                    )
                 elif constraint.sense == constants.LpConstraintGE:
-                    relation = gurobipy.GRB.GREATER_EQUAL
+                    constraint.solverConstraint = lp.solverModel.addConstr(
+                        expr >= -constraint.constant, name=name
+                    )
                 elif constraint.sense == constants.LpConstraintEQ:
-                    relation = gurobipy.GRB.EQUAL
+                    constraint.solverConstraint = lp.solverModel.addConstr(
+                        expr == -constraint.constant, name=name
+                    )
                 else:
                     raise PulpSolverError("Detected an invalid constraint type")
-                constraint.solverConstraint = lp.solverModel.addConstr(
-                    expr, relation, -constraint.constant, name
-                )
             lp.solverModel.update()
 
         def actualSolve(self, lp, callback=None):
@@ -267,7 +371,7 @@ def actualResolve(self, lp, callback=None):
             for constraint in lp.constraints.values():
                 if constraint.modified:
                     constraint.solverConstraint.setAttr(
-                        gurobipy.GRB.Attr.RHS, -constraint.constant
+                        gp.GRB.Attr.RHS, -constraint.constant
                     )
             lp.solverModel.update()
             self.callSolver(lp, callback=callback)

pulp/tests/run_tests.py

@@ -1,6 +1,6 @@
 import unittest
 import pulp
-from pulp.tests import test_pulp, test_examples
+from pulp.tests import test_pulp, test_examples, test_gurobipy_env
 
 
 def pulpTestAll(test_docs=False):
@@ -19,6 +19,9 @@ def get_test_suite(test_docs=False):
     # we get suite with all PuLP tests
     pulp_solver_tests = loader.loadTestsFromModule(test_pulp)
     suite_all.addTests(pulp_solver_tests)
+    # Add tests for gurobipy env
+    gurobipy_env = loader.loadTestsFromModule(test_gurobipy_env)
+    suite_all.addTests(gurobipy_env)
     # We add examples and docs tests
     if test_docs:
         docs_examples = loader.loadTestsFromTestCase(test_examples.Examples_DocsTests)