Issue 1133 - Fixes flaky Visdom tests

ignite/contrib/handlers/visdom_logger.py

@@ -454,7 +454,7 @@ class VisdomLogger(BaseLogger):
 
     """
 
-    def __init__(self, server=None, port=None, num_workers=1, **kwargs):
+    def __init__(self, server=None, port=None, num_workers=1, raise_exceptions=True, **kwargs):
         try:
             import visdom
         except ImportError:
@@ -491,11 +491,11 @@ def __init__(self, server=None, port=None, num_workers=1, **kwargs):
             password = os.environ.get("VISDOM_PASSWORD", None)
             kwargs["password"] = password
 
-        self.vis = visdom.Visdom(server=server, port=port, **kwargs)
+        self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs)
 
-        if not self.vis.check_connection():
+        if not self.vis.offline and not self.vis.check_connection():
             raise RuntimeError(
-                "Failed to connect to Visdom server at {}. " "Did you run python -m visdom.server ?".format(server)
+                "Failed to connect to Visdom server at {}. Did you run python -m visdom.server ?".format(server)
             )
 
         self.executor = _DummyExecutor()
@@ -508,8 +508,8 @@ def _save(self):
         self.vis.save([self.vis.env])
 
     def close(self):
-        self.vis = None
         self.executor.shutdown()
+        self.vis = None
 
     def _create_output_handler(self, *args, **kwargs):
         return OutputHandler(*args, **kwargs)

tests/ignite/contrib/conftest.py

@@ -1,21 +1,63 @@
+import random
+
 import pytest
 
 
 @pytest.fixture()
-def visdom_server():
+def visdom_offline_logfile(dirname):
 
     import os
-    import signal
-    import subprocess
-    import time
 
-    from visdom.server import download_scripts
+    log_file = os.path.join(dirname, "logs.visdom")
+    yield log_file
+
+
+vd_hostname = None
+vd_port = None
+vd_server_process = None
+
+
+@pytest.fixture()
+def visdom_server():
+    # Start Visdom server once and stop it with visdom_server_stop
+    global vd_hostname, vd_port, vd_server_process
+
+    if vd_server_process is None:
+
+        import subprocess
+        import time
+
+        from visdom import Visdom
+        from visdom.server import download_scripts
 
-    download_scripts()
+        download_scripts()
+
+        vd_hostname = "localhost"
+        vd_port = random.randint(8089, 8887)
+
+        try:
+            vis = Visdom(server=vd_hostname, port=vd_port, raise_exceptions=True)
+        except ConnectionError:
+            pass
+
+        vd_server_process = subprocess.Popen(
+            ["python", "-m", "visdom.server", "--hostname", vd_hostname, "-port", str(vd_port)]
+        )
+        time.sleep(5)
+
+        vis = Visdom(server=vd_hostname, port=vd_port)
+        assert vis.check_connection()
+        vis.close()
+
+    yield (vd_hostname, vd_port)
+
+
+@pytest.fixture()
+def visdom_server_stop():
+
+    yield None
+
+    import time
 
-    hostname = "localhost"
-    port = 8098
-    p = subprocess.Popen("visdom --hostname {} -port {}".format(hostname, port), shell=True, preexec_fn=os.setsid)
-    time.sleep(5)
-    yield (hostname, port)
-    os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    vd_server_process.kill()
+    time.sleep(2)

tests/ignite/contrib/engines/test_common.py

@@ -403,10 +403,10 @@ def test_setup_tb_logging(dirname):
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
-def test_setup_visdom_logging(visdom_server):
+def test_setup_visdom_logging(visdom_offline_logfile):
     vis_logger = _test_setup_logging(
         setup_logging_fn=setup_visdom_logging,
-        kwargs_dict={"server": visdom_server[0], "port": str(visdom_server[1])},
+        kwargs_dict={"offline": True, "log_to_filename": visdom_offline_logfile},
         output_handler_cls=handlers.visdom_logger.OutputHandler,
         opt_params_handler_cls=handlers.visdom_logger.OptimizerParamsHandler,
         with_eval=False,
@@ -416,7 +416,7 @@ def test_setup_visdom_logging(visdom_server):
 
     vis_logger = _test_setup_logging(
         setup_logging_fn=setup_visdom_logging,
-        kwargs_dict={"server": visdom_server[0], "port": str(visdom_server[1])},
+        kwargs_dict={"offline": True, "log_to_filename": visdom_offline_logfile},
         output_handler_cls=handlers.visdom_logger.OutputHandler,
         opt_params_handler_cls=handlers.visdom_logger.OptimizerParamsHandler,
         with_eval=True,

tests/ignite/contrib/handlers/test_lr_finder.py

@@ -199,14 +199,13 @@ def test_num_iter_is_not_enough(lr_finder, to_save, dummy_engine, dataloader):
         assert dummy_engine.state.iteration == len(dataloader)
 
 
-def test_detach_terminates(lr_finder, to_save, dummy_engine, dataloader):
+def test_detach_terminates(lr_finder, to_save, dummy_engine, dataloader, recwarn):
     with lr_finder.attach(dummy_engine, to_save, end_lr=100, diverge_th=2) as trainer_with_finder:
-        with pytest.warns(None) as record:
-            trainer_with_finder.run(dataloader)
-            assert len(record) == 0
+        trainer_with_finder.run(dataloader)
 
     dummy_engine.run(dataloader, max_epochs=3)
     assert dummy_engine.state.epoch == 3
+    assert len(recwarn) == 0
 
 
 def test_lr_suggestion(lr_finder, to_save, dummy_engine, dataloader):

tests/ignite/contrib/handlers/test_visdom_logger.py

@@ -788,18 +788,16 @@ def _test(tag=None):
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
 def test_integration_no_server():
 
-    with pytest.warns(
-        PendingDeprecationWarning, match="Visdom is eventually changing to default to raising exceptions"
-    ):
-        with pytest.raises(RuntimeError, match="Failed to connect to Visdom server"):
-            VisdomLogger()
+    with pytest.raises(ConnectionError, match="Error connecting to Visdom server"):
+        VisdomLogger()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
 def test_logger_init_hostname_port(visdom_server):
     # Explicit hostname, port
     vd_logger = VisdomLogger(server=visdom_server[0], port=visdom_server[1], num_workers=0)
     assert "main" in vd_logger.vis.get_env_list()
+    vd_logger.close()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
@@ -811,6 +809,7 @@ def test_logger_init_env_vars(visdom_server):
     os.environ["VISDOM_PORT"] = str(visdom_server[1])
     vd_logger = VisdomLogger(server=visdom_server[0], port=visdom_server[1], num_workers=0)
     assert "main" in vd_logger.vis.get_env_list()
+    vd_logger.close()
 
 
 def _parse_content(content):
@@ -852,6 +851,7 @@ def update_fn(engine, batch):
     x_vals, y_vals = data["x"], data["y"]
     assert all([int(x) == x_true for x, x_true in zip(x_vals, list(range(1, n_epochs * len(data) + 1)))])
     assert all([y == y_true for y, y_true in zip(y_vals, losses)])
+    vd_logger.close()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
@@ -892,7 +892,7 @@ def update_fn(engine, batch):
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
-def test_integration_with_executor_as_context_manager(visdom_server):
+def test_integration_with_executor_as_context_manager(visdom_server, visdom_server_stop):
 
     n_epochs = 5
     data = list(range(50))