From 0029c447bac8b1fae16479320619033c8b29c7cf Mon Sep 17 00:00:00 2001 From: Francesco Nuzzo Date: Sun, 27 Feb 2022 20:40:30 +0100 Subject: [PATCH 1/5] async capture solver output --- pulp/apis/coin_api.py | 79 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/pulp/apis/coin_api.py b/pulp/apis/coin_api.py index da29f908..3f6b9b06 100644 --- a/pulp/apis/coin_api.py +++ b/pulp/apis/coin_api.py @@ -24,6 +24,9 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.""" +import asyncio +from io import TextIOWrapper +import sys from .core import LpSolver_CMD, LpSolver, subprocess, PulpSolverError, clock, log from .core import cbc_path, pulp_cbc_path, coinMP_path, devnull import os @@ -176,33 +179,77 @@ def solve_CBC(self, lp, use_mps=True): cmds += "initialSolve " cmds += "printingOptions all " cmds += "solution " + tmpSol + " " - if self.msg: - pipe = None - else: - pipe = open(os.devnull, "w") + + # self.msg = 1 + # self.optionsDict["logPath"] = "this.log" + logPath = self.optionsDict.get("logPath") - if logPath: - if self.msg: - warnings.warn( - "`logPath` argument replaces `msg=1`. The output will be redirected to the log file." - ) - pipe = open(self.optionsDict["logPath"], "w") + log.debug(self.path + cmds) args = [] args.append(self.path) args.extend(cmds[1:].split()) - cbc = subprocess.Popen(args, stdout=pipe, stderr=pipe, stdin=devnull) - if cbc.wait() != 0: - if pipe: - pipe.close() + # https://lyceum-allotments.github.io/2017/03/python-and-pipes-part-6-multiple-subprocesses-and-pipes/ + + class SubProcessRunner: + def __init__(self, print_to_stdout, logPath=None): + self.loop = self.get_event_loop() + self.output = [] + self.full_output = None + self.print_to_stdout = print_to_stdout + self.logPath = logPath or os.devnull + + @staticmethod + def get_event_loop(): + if sys.platform == "win32": + loop = asyncio.ProactorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_running_loop() + return loop + + async def watch(self, stream): + with open(self.logPath, "w") as f: + async for line in stream: + line = line.decode().strip() + self.output.append(line) + + if self.print_to_stdout: + print(line) + f.write(line + "\n") + + async def run_sub(self, cmd): + p = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + await asyncio.gather(self.watch(p.stdout)) + await p.wait() + return p.returncode + + def run(self, args): + return_code = self.loop.run_until_complete(self.run_sub(args)) + self.full_output = "\n".join(self.output) + self.loop.close() + return return_code + + sub_process_runner = SubProcessRunner(print_to_stdout=self.msg, logPath=logPath) + return_code = sub_process_runner.run(args) + + if return_code != 0: + print(return_code) raise PulpSolverError( "Pulp: Error while trying to execute, use msg=True for more details" + self.path + + "return_code={}\noutput={}".format( + return_code, sub_process_runner.full_output + ) ) - if pipe: - pipe.close() + if not os.path.exists(tmpSol): raise PulpSolverError("Pulp: Error while executing " + self.path) + # Problem: output is read from file + # readsol_MPS requires multiple dispatch or conditional reading with + # different functions ( status, values, From 6201443076a23a6679e4235b3b1fea14a26e3b61 Mon Sep 17 00:00:00 2001 From: Francesco Nuzzo Date: Wed, 2 Mar 2022 21:56:07 +0100 Subject: [PATCH 2/5] Save output --- pulp/apis/coin_api.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pulp/apis/coin_api.py b/pulp/apis/coin_api.py index 3f6b9b06..9dd0f1c7 100644 --- a/pulp/apis/coin_api.py +++ b/pulp/apis/coin_api.py @@ -180,19 +180,15 @@ def solve_CBC(self, lp, use_mps=True): cmds += "printingOptions all " cmds += "solution " + tmpSol + " " - # self.msg = 1 - # self.optionsDict["logPath"] = "this.log" - logPath = self.optionsDict.get("logPath") log.debug(self.path + cmds) args = [] args.append(self.path) args.extend(cmds[1:].split()) - # https://lyceum-allotments.github.io/2017/03/python-and-pipes-part-6-multiple-subprocesses-and-pipes/ class SubProcessRunner: - def __init__(self, print_to_stdout, logPath=None): + def __init__(self, print_to_stdout: bool, logPath: str = None): self.loop = self.get_event_loop() self.output = [] self.full_output = None @@ -245,11 +241,10 @@ def run(self, args): ) ) + self.solver_output = sub_process_runner.full_output + if not os.path.exists(tmpSol): raise PulpSolverError("Pulp: Error while executing " + self.path) - # Problem: output is read from file - # readsol_MPS requires multiple dispatch or conditional reading with - # different functions ( status, values, From b8d7d1c6afd206cdfa3ec28cf2a8371ea46fd46a Mon Sep 17 00:00:00 2001 From: Francesco Nuzzo Date: Tue, 8 Mar 2022 13:50:05 +0100 Subject: [PATCH 3/5] remove unused imports --- pulp/apis/coin_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulp/apis/coin_api.py b/pulp/apis/coin_api.py index 9dd0f1c7..b35c5bab 100644 --- a/pulp/apis/coin_api.py +++ b/pulp/apis/coin_api.py @@ -25,9 +25,8 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.""" import asyncio -from io import TextIOWrapper import sys -from .core import LpSolver_CMD, LpSolver, subprocess, PulpSolverError, clock, log +from .core import LpSolver_CMD, LpSolver, PulpSolverError, clock, log from .core import cbc_path, pulp_cbc_path, coinMP_path, devnull import os from .. import constants From 5f307077ab1382b2d4246b53e29bf5d39499717f Mon Sep 17 00:00:00 2001 From: Francesco Nuzzo Date: Sat, 12 Mar 2022 13:01:15 +0100 Subject: [PATCH 4/5] fix get_event_loop --- pulp/apis/coin_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulp/apis/coin_api.py b/pulp/apis/coin_api.py index b35c5bab..bd9ed740 100644 --- a/pulp/apis/coin_api.py +++ b/pulp/apis/coin_api.py @@ -198,9 +198,9 @@ def __init__(self, print_to_stdout: bool, logPath: str = None): def get_event_loop(): if sys.platform == "win32": loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) else: - loop = asyncio.get_running_loop() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) return loop async def watch(self, stream): From 303447708594998613d1a6f260ac75a1d745b4a4 Mon Sep 17 00:00:00 2001 From: Francesco Nuzzo Date: Sat, 12 Mar 2022 20:22:44 +0100 Subject: [PATCH 5/5] Remove type annotations for compatibility with python 2.7 --- pulp/apis/coin_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulp/apis/coin_api.py b/pulp/apis/coin_api.py index bd9ed740..e458ce2a 100644 --- a/pulp/apis/coin_api.py +++ b/pulp/apis/coin_api.py @@ -187,7 +187,7 @@ def solve_CBC(self, lp, use_mps=True): args.extend(cmds[1:].split()) class SubProcessRunner: - def __init__(self, print_to_stdout: bool, logPath: str = None): + def __init__(self, print_to_stdout, logPath=None): self.loop = self.get_event_loop() self.output = [] self.full_output = None