Make tests robust and independent

.github/workflows/pytest.yaml

@@ -135,13 +135,12 @@ jobs:
       shell: Rscript {0}
 
     - name: Run test suite using pytest
-    # FIXME: Use --numprocesses=auto once flaky tests are fixed
       run: |
         pytest ixmp \
           -m "not performance" \
           --color=yes -rA --verbose \
           --cov-report=xml \
-          --numprocesses=1
+          --numprocesses=auto --dist=loadgroup
       shell: bash
 
     - name: Upload test coverage to Codecov.io

ixmp/backend/jdbc.py

@@ -728,6 +728,7 @@ def del_ts(self, ts):
 
         # Aggressively free memory
         self.gc()
+        self.jindex.pop(ts, None)
 
     def check_out(self, ts, timeseries_only):
         with _handle_jexception():

ixmp/core/platform.py

@@ -214,6 +214,7 @@ def export_timeseries_data(
             - subannual
             - year
             - value
+
         default : bool, optional
             :obj:`True` to include only TimeSeries versions marked as default.
         model: str, optional

ixmp/testing/data.py

@@ -1,18 +1,42 @@
 # Methods are in alphabetical order
 from itertools import product
 from math import ceil
-from typing import Any, List
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 import genno
 import numpy as np
 import pandas as pd
 import pint
+import pytest
 
 from ixmp import Platform, Scenario, TimeSeries
 from ixmp.backend import IAMC_IDX
 
+if TYPE_CHECKING:
+    from typing import TypedDict
+
+    class ScenarioIdentifiers(TypedDict):
+        """Identifiers of a Scenario.
+
+        Used only for type checking in this file, so version is omitted.
+        """
+
+        model: str
+        scenario: str
+
+    class ScenarioKwargs(TypedDict, total=False):
+        """Keyword arguments to Scenario.__init__()."""
+
+        model: str
+        scenario: str
+        version: str
+        scheme: str
+        annotation: str
+        with_data: Optional[bool]
+
+
 #: Common (model name, scenario name) pairs for testing.
-SCEN = {
+SCEN: Dict[str, "ScenarioIdentifiers"] = {
     "dantzig": dict(model="canning problem", scenario="standard"),
     "h2g2": dict(model="Douglas Adams", scenario="Hitchhiker"),
 }
@@ -158,7 +182,12 @@ def add_test_data(scen: Scenario):
     return t, t_foo, t_bar, x
 
 
-def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scenario:
+def make_dantzig(
+    mp: Platform,
+    solve: bool = False,
+    quiet: bool = False,
+    request: Optional["pytest.FixtureRequest"] = None,
+) -> Scenario:
     """Return :class:`ixmp.Scenario` of Dantzig's canning/transport problem.
 
     Parameters
@@ -169,6 +198,8 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen
         If :obj:`True`. then solve the scenario before returning. Default :obj:`False`.
     quiet : bool, optional
         If :obj:`True`, suppress console output when solving.
+    request : :class:`pytest.FixtureRequest`, optional
+        If present, use for a distinct scenario name for each test.
 
     Returns
     -------
@@ -178,38 +209,39 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen
     --------
     .DantzigModel
     """
-    # add custom units and region for timeseries data
+    # Add custom units and region for time series data
     try:
         mp.add_unit("USD/km")
     except Exception:
         # Unit already exists. Pending bugfix from zikolach
         pass
     mp.add_region("DantzigLand", "country")
 
-    # Initialize a new Scenario, and use the DantzigModel class' initialize()
-    # method to populate it
-    annot = "Dantzig's transportation problem for illustration and testing"
-    scen = Scenario(
-        mp,
-        **models["dantzig"],  # type: ignore [arg-type]
+    # Initialize a new Scenario, and use the DantzigModel class' initialize() method to
+    # populate it
+    args: "ScenarioKwargs" = dict(
+        **models["dantzig"],
         version="new",
-        annotation=annot,
+        annotation="Dantzig's transportation problem for illustration and testing",
         scheme="dantzig",
         with_data=True,
     )
+    # Use a distinct scenario name for a particular test
+    args.update({"scenario": request.node.name} if request else {})
+
+    scen = Scenario(mp, **args)
 
-    # commit the scenario
+    # Commit the scenario
     scen.commit("Import Dantzig's transport problem for testing.")
 
-    # set this new scenario as the default version for the model/scenario name
+    # Set this new scenario as the default version for the model/scenario name
     scen.set_as_default()
 
     if solve:
         # Solve the model using the GAMS code provided in the `tests` folder
         scen.solve(model="dantzig", case="transport_standard", quiet=quiet)
 
-    # add timeseries data for testing `clone(keep_solution=False)`
-    # and `remove_solution()`
+    # Add time series data for testing clone(keep_solution=False) and remove_solution()
     scen.check_out(timeseries_only=True)
     scen.add_timeseries(HIST_DF, meta=True)
     scen.add_timeseries(INP_DF)

ixmp/testing/jupyter.py

@@ -48,16 +48,6 @@ def run_notebook(nb_path, tmp_path, env=None, **kwargs):
     import nbformat
     from nbclient import NotebookClient
 
-    # Workaround for https://github.com/jupyter/nbclient/issues/85
-    if (
-        sys.version_info[0] == 3
-        and sys.version_info[1] >= 8
-        and sys.platform.startswith("win")
-    ):
-        import asyncio
-
-        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-
     # Read the notebook
     nb = nbformat.read(nb_path, as_version=4)
 

ixmp/tests/backend/test_base.py

@@ -135,14 +135,14 @@ def test_cache_invalidate(self, test_mp):
 
         backend.cache_invalidate(ts, "par", "baz", dict(x=["x1", "x2"], y=["y1", "y2"]))
 
-    def test_del_ts(self, test_mp):
+    def test_del_ts(self, test_mp, request):
         """Test CachingBackend.del_ts()."""
         # Since CachingBackend is an abstract class, test it via JDBCBackend
-        backend = test_mp._backend
+        backend: CachingBackend = test_mp._backend  # type: ignore
         cache_size_pre = len(backend._cache)
 
         # Load data, thereby adding to the cache
-        s = make_dantzig(test_mp)
+        s = make_dantzig(test_mp, request=request)
         s.par("d")
 
         # Cache size has increased
@@ -155,7 +155,7 @@ def test_del_ts(self, test_mp):
             s.__del__()  # Force deletion of cached objects associated with `s`
 
         # Delete the object; associated cache is freed
-        del s
+        backend.del_ts(s)
 
         # Objects were invalidated/removed from cache
         assert cache_size_pre == len(backend._cache)