Change multiprocessing to multithreading, fix CI, add tests

.github/workflows/ci.yml

@@ -2,6 +2,8 @@ name: Lint and test
 
 on:
   push:
+    branches:
+      - '**'
     paths-ignore:
       - 'docs/**'
       - '**.md'

README.md

@@ -88,6 +88,3 @@ This will also install [flask](https://pypi.org/project/Flask/).
 
 # Documentation
 More documentation can be found in [docs](https://github.com/kvnglb/netargparse/tree/main/docs).
-
-# TODO
-- Windows support

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "netargparse"
-version = "0.1.1"
+version = "0.1.2"
 description = "Enhance ArgumentParser with a TCP-based API for argument handling."
 authors = ["Kevin Golob <151143873+kvnglb@users.noreply.github.com>"]
 readme = "README.md"
@@ -13,7 +13,7 @@ classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
     "License :: OSI Approved :: BSD License",
-    "Operating System :: POSIX :: Linux",
+    "Operating System :: OS Independent",
     "Programming Language :: Python :: 3 :: Only",
     "Typing :: Typed"
 ]

src/netargparse/server.py

@@ -134,26 +134,20 @@ class HttpServer:
 
     Attributes
     ----------
-    p_get_r : multiprocessing.connection.Connection
-        The ending of the pipe, that receives the message received by the client
-        from the flask daemon process.
-    p_get_s : multiprocessing.connection.Connection
-        The ending of the pipe, that sends the message received by the client
-        to the main process.
-    p_send_r : multiprocessing.connection.Connection
-        The ending of the pipe, that receives the message from `NetArgumentParser`
-        from the main process.
-    p_send_s : multiprocessing.connection.Connection
-        The ending of the pipe, that sends the message from `NetArgumentParser`
-        to the flask daemon process.
+    q_get : queue.Queue
+        The queue, that sends and receives the message received by the client
+        from the flask daemon thread to the main thread.
+    q_send : queue.Queue
+        The queue, that sends and receives the message from `NetArgumentParser`
+        from the main thread to the flask daemon thread.
 
     """
 
     def __init__(self, ip: str, port: int) -> None:
-        """Initialize flask as daemon process to accept http get requests.
+        """Initialize flask as daemon thread to accept http get requests.
 
-        Flask is started as daemon process and the url parameters are sent
-        to the main process through pipes for converting them into the argument
+        Flask is started as daemon thread and the url parameters are sent
+        to the main thread through queues for converting them into the argument
         string, that is needed for the main `parser`.
 
         Parameters
@@ -164,16 +158,16 @@ def __init__(self, ip: str, port: int) -> None:
             The port, where the socket should listen.
 
         """
-        from multiprocessing import Pipe, Process
-        from multiprocessing.connection import Connection
+        from queue import Queue
+        from threading import Thread
 
         import flask
 
-        self.p_get_r, self.p_get_s = Pipe(False)
-        self.p_send_r, self.p_send_s = Pipe(False)
+        self.q_get = Queue(maxsize=1)  # type: Queue
+        self.q_send = Queue(maxsize=1)  # type: Queue
 
-        def serve(p_get_s: Connection, p_send_r: Connection) -> None:
-            """Daemon process, that is running flask."""
+        def serve(q_get: Queue, q_send: Queue) -> None:
+            """Daemon thread, that is running flask."""
             app = flask.Flask(__name__)
 
             def msg_handler(autoformat: bool, response: t.Union[dict, str],
@@ -212,37 +206,37 @@ def msg_handler(autoformat: bool, response: t.Union[dict, str],
             def http_get() -> flask.wrappers.Response:
                 """Route for json response."""
                 d = dict(flask.request.args)
-                p_get_s.send(d)
-                r = p_send_r.recv()  # type: tuple[t.Any, t.Any, t.Any]
+                q_get.put(d)
+                r = q_send.get()  # type: tuple[t.Any, t.Any, t.Any]
                 return flask.Response(msg_handler(*r, MessageJson),
                                       mimetype="application/json")
 
             @app.route("/xml")
             def http_get_xml() -> flask.wrappers.Response:
                 """Route for xml response."""
                 d = dict(flask.request.args)
-                p_get_s.send(d)
-                r = p_send_r.recv()  # type: tuple[t.Any, t.Any, t.Any]
+                q_get.put(d)
+                r = q_send.get()  # type: tuple[t.Any, t.Any, t.Any]
                 return flask.Response(msg_handler(*r, MessageXml),
                                       mimetype="application/xml")
 
             app.run(ip, port)
 
-        proc_serve = Process(target=serve, args=(self.p_get_s, self.p_send_r), daemon=True)
-        proc_serve.start()
+        thrd_serve = Thread(target=serve, args=(self.q_get, self.q_send), daemon=True)
+        thrd_serve.start()
 
     def get_msg(self) -> str:
         """Receive the message that was sent from the client to the http server.
 
-        Receive the message as dict from the daemon process via the pipe and return
-        the corresponding argument string for the main `parser`.
+        Receive the message as dict from the daemon thread via the queue and
+        return the corresponding argument string for the main `parser`.
 
         Returns
         -------
         Argument string for main `parser`.
 
         """
-        d = self.p_get_r.recv()
+        d = self.q_get.get()
         return Message.dict_to_argstring(d)
 
     def send_msg(self, autoformat: bool, response: t.Union[dict, str],
@@ -265,6 +259,6 @@ def send_msg(self, autoformat: bool, response: t.Union[dict, str],
 
         """
         try:
-            self.p_send_s.send((autoformat, response, exception))
+            self.q_send.put((autoformat, response, exception))
         except Exception as e:
             print(e)

tests/test_netargparse.py

@@ -11,6 +11,8 @@
 
 def tcp_socket_no_autoformat():
     def main(args):
+        if args.var_str == "damn":
+            return args.var_int / 0
         return f'"{args}"'
 
     nap = NetArgumentParser()
@@ -21,6 +23,8 @@ def main(args):
 
 def tcp_socket_autoformat():
     def main(args):
+        if args.var_str == "damn":
+            return args.var_int / 0
         return vars(args)
 
     nap = NetArgumentParser()
@@ -31,6 +35,8 @@ def main(args):
 
 def http_no_autoformat():
     def main(args):
+        if args.var_str == "damn":
+            return args.var_int / 0
         return f'"{args}"'
 
     nap = NetArgumentParser()
@@ -41,6 +47,8 @@ def main(args):
 
 def http_autoformat():
     def main(args):
+        if args.var_str == "damn":
+            return args.var_int / 0
         return vars(args)
 
     nap = NetArgumentParser()
@@ -82,6 +90,7 @@ def assertResponse(self, resp, resp_type):
         except Exception as e:
             self.fail(e)
 
+    # Plain xml, no autoformat
     def test_plain_xml_na_valid_tx(self):
         ans = s_tcp_na.txrx(b"<nap><__var_str>value</__var_str><__var_int>2</__var_int></nap>")
         self.assertEqual(ans, b"<nap><response>\"Namespace(var_str='value', var_int=2, var_true=False, _cmd='nap')\"</response><exception></exception><finished>1</finished></nap>")
@@ -97,6 +106,12 @@ def test_plain_xml_na_invalid_int(self):
         self.assertEqual(ans, b"<nap><response></response><exception>argument --var_int: invalid int value: '2.2'</exception><finished>1</finished></nap>")
         self.assertResponse(ans, "xml")
 
+    def test_plain_xml_na_func_exc(self):
+        ans = s_tcp_na.txrx(b"<nap><__var_str>damn</__var_str><__var_int>5</__var_int></nap>")
+        self.assertEqual(ans, b"<nap><response></response><exception>division by zero</exception><finished>1</finished></nap>")
+        self.assertResponse(ans, "xml")
+
+    # Plain json, no autoformat
     def test_plain_json_na_valid_tx(self):
         ans = s_tcp_na.txrx(b'{"--var_str": "value", "--var_int": "2"}')
         self.assertEqual(ans, b'{"response": "Namespace(var_str=\'value\', var_int=2, var_true=False, _cmd=\'nap\')", "exception": "", "finished": 1}')
@@ -112,6 +127,12 @@ def test_plain_json_na_invalid_int(self):
         self.assertEqual(ans, b'{"response": "", "exception": "argument --var_int: invalid int value: \'2.2\'", "finished": 1}')
         self.assertResponse(ans, "json")
 
+    def test_plain_json_na_func_exc(self):
+        ans = s_tcp_na.txrx(b'{"--var_str": "damn", "--var_int": "5"}')
+        self.assertEqual(ans, b'{"response": "", "exception": "division by zero", "finished": 1}')
+        self.assertResponse(ans, "json")
+
+    # Plain xml, autoformat
     def test_plain_xml_a_valid_tx(self):
         ans = s_tcp_a.txrx(b"<nap><__var_str>value</__var_str><__var_int>2</__var_int></nap>")
         self.assertEqual(ans, b"<nap><response><var_str>value</var_str><var_int>2</var_int><var_true>False</var_true><_cmd>nap</_cmd></response><exception></exception><finished>1</finished></nap>")
@@ -127,6 +148,12 @@ def test_plain_xml_a_invalid_int(self):
         self.assertEqual(ans, b"<nap><response></response><exception>argument --var_int: invalid int value: '2.2'</exception><finished>1</finished></nap>")
         self.assertResponse(ans, "xml")
 
+    def test_plain_xml_a_func_exc(self):
+        ans = s_tcp_a.txrx(b"<nap><__var_str>damn</__var_str><__var_int>5</__var_int></nap>")
+        self.assertEqual(ans, b"<nap><response></response><exception>division by zero</exception><finished>1</finished></nap>")
+        self.assertResponse(ans, "xml")
+
+    # Plain json, autoformat
     def test_plain_json_a_valid_tx(self):
         ans = s_tcp_a.txrx(b'{"--var_str": "value", "--var_int": "2"}')
         self.assertEqual(ans, b'{"response": {"var_str": "value", "var_int": 2, "var_true": false, "_cmd": "nap"}, "exception": "", "finished": 1}')
@@ -142,6 +169,12 @@ def test_plain_json_a_invalid_int(self):
         self.assertEqual(ans, b'{"response": "", "exception": "argument --var_int: invalid int value: \'2.2\'", "finished": 1}')
         self.assertResponse(ans, "json")
 
+    def test_plain_json_a_func_exc(self):
+        ans = s_tcp_a.txrx(b'{"--var_str": "damn", "--var_int": "5"}')
+        self.assertEqual(ans, b'{"response": "", "exception": "division by zero", "finished": 1}')
+        self.assertResponse(ans, "json")
+
+    # HTTP, json resp, no autoformat
     def test_http_json_na_valid_tx(self):
         ans = s_http_na.txrx("/?--var_str=value&--var_int=2")
         self.assertEqual(ans, '{"response": "Namespace(var_str=\'value\', var_int=2, var_true=False, _cmd=\'nap\')", "exception": "", "finished": 1}')
@@ -157,6 +190,12 @@ def test_http_json_na_invalid_int(self):
         self.assertEqual(ans, '{"response": "", "exception": "argument --var_int: invalid int value: \'2.2\'", "finished": 1}')
         self.assertResponse(ans, "json")
 
+    def test_http_json_na_func_exc(self):
+        ans = s_http_na.txrx("/?--var_str=damn&--var_int=5")
+        self.assertEqual(ans, '{"response": "", "exception": "division by zero", "finished": 1}')
+        self.assertResponse(ans, "json")
+
+    # HTTP, xml resp, no autoformat
     def test_http_xml_na_valid_tx(self):
         ans = s_http_na.txrx("/xml?--var_str=value&--var_int=2")
         self.assertEqual(ans, "<nap><response>\"Namespace(var_str='value', var_int=2, var_true=False, _cmd='nap')\"</response><exception></exception><finished>1</finished></nap>")
@@ -172,6 +211,12 @@ def test_http_xml_na_invalid_int(self):
         self.assertEqual(ans, "<nap><response></response><exception>argument --var_int: invalid int value: '2.2'</exception><finished>1</finished></nap>")
         self.assertResponse(ans, "xml")
 
+    def test_http_xml_na_func_exc(self):
+        ans = s_http_na.txrx("/xml?--var_str=damn&--var_int=5")
+        self.assertEqual(ans, "<nap><response></response><exception>division by zero</exception><finished>1</finished></nap>")
+        self.assertResponse(ans, "xml")
+
+    # HTTP, json resp, autoformat
     def test_http_json_a_valid_tx(self):
         ans = s_http_a.txrx("/?--var_str=value&--var_int=2")
         self.assertEqual(ans, '{"response": {"var_str": "value", "var_int": 2, "var_true": false, "_cmd": "nap"}, "exception": "", "finished": 1}')
@@ -187,6 +232,12 @@ def test_http_json_a_invalid_int(self):
         self.assertEqual(ans, '{"response": "", "exception": "argument --var_int: invalid int value: \'2.2\'", "finished": 1}')
         self.assertResponse(ans, "json")
 
+    def test_http_json_a_func_exc(self):
+        ans = s_http_a.txrx("/?--var_str=damn&--var_int=5")
+        self.assertEqual(ans, '{"response": "", "exception": "division by zero", "finished": 1}')
+        self.assertResponse(ans, "json")
+
+    # HTTP, xml resp, autoformat
     def test_http_xml_a_valid_tx(self):
         ans = s_http_a.txrx("/xml?--var_str=value&--var_int=2")
         self.assertEqual(ans, "<nap><response><var_str>value</var_str><var_int>2</var_int><var_true>False</var_true><_cmd>nap</_cmd></response><exception></exception><finished>1</finished></nap>")
@@ -202,6 +253,11 @@ def test_http_xml_a_invalid_int(self):
         self.assertEqual(ans, "<nap><response></response><exception>argument --var_int: invalid int value: '2.2'</exception><finished>1</finished></nap>")
         self.assertResponse(ans, "xml")
 
+    def test_http_xml_a_func_exc(self):
+        ans = s_http_a.txrx("/xml?--var_str=damn&--var_int=5")
+        self.assertEqual(ans, "<nap><response></response><exception>division by zero</exception><finished>1</finished></nap>")
+        self.assertResponse(ans, "xml")
+
 
 if __name__ == "__main__":
     for t in [tcp_socket_no_autoformat, tcp_socket_autoformat, http_no_autoformat, http_autoformat]: