diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 8df2bee22..2f0fa21d2 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -135,13 +135,12 @@ jobs: shell: Rscript {0} - name: Run test suite using pytest - # FIXME: Use --numprocesses=auto once flaky tests are fixed run: | pytest ixmp \ -m "not performance" \ --color=yes -rA --verbose \ --cov-report=xml \ - --numprocesses=1 + --numprocesses=auto --dist=loadgroup shell: bash - name: Upload test coverage to Codecov.io diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 35d471385..805b9054b 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -728,6 +728,7 @@ def del_ts(self, ts): # Aggressively free memory self.gc() + self.jindex.pop(ts, None) def check_out(self, ts, timeseries_only): with _handle_jexception(): diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 10b770b27..ce7021858 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -214,6 +214,7 @@ def export_timeseries_data( - subannual - year - value + default : bool, optional :obj:`True` to include only TimeSeries versions marked as default. model: str, optional diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 877c64f6e..9de18009c 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -1,18 +1,42 @@ # Methods are in alphabetical order from itertools import product from math import ceil -from typing import Any, List +from typing import TYPE_CHECKING, Any, Dict, List, Optional import genno import numpy as np import pandas as pd import pint +import pytest from ixmp import Platform, Scenario, TimeSeries from ixmp.backend import IAMC_IDX +if TYPE_CHECKING: + from typing import TypedDict + + class ScenarioIdentifiers(TypedDict): + """Identifiers of a Scenario. + + Used only for type checking in this file, so version is omitted. + """ + + model: str + scenario: str + + class ScenarioKwargs(TypedDict, total=False): + """Keyword arguments to Scenario.__init__().""" + + model: str + scenario: str + version: str + scheme: str + annotation: str + with_data: Optional[bool] + + #: Common (model name, scenario name) pairs for testing. -SCEN = { +SCEN: Dict[str, "ScenarioIdentifiers"] = { "dantzig": dict(model="canning problem", scenario="standard"), "h2g2": dict(model="Douglas Adams", scenario="Hitchhiker"), } @@ -158,7 +182,12 @@ def add_test_data(scen: Scenario): return t, t_foo, t_bar, x -def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scenario: +def make_dantzig( + mp: Platform, + solve: bool = False, + quiet: bool = False, + request: Optional["pytest.FixtureRequest"] = None, +) -> Scenario: """Return :class:`ixmp.Scenario` of Dantzig's canning/transport problem. Parameters @@ -169,6 +198,8 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen If :obj:`True`. then solve the scenario before returning. Default :obj:`False`. quiet : bool, optional If :obj:`True`, suppress console output when solving. + request : :class:`pytest.FixtureRequest`, optional + If present, use for a distinct scenario name for each test. Returns ------- @@ -178,7 +209,7 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen -------- .DantzigModel """ - # add custom units and region for timeseries data + # Add custom units and region for time series data try: mp.add_unit("USD/km") except Exception: @@ -186,30 +217,31 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen pass mp.add_region("DantzigLand", "country") - # Initialize a new Scenario, and use the DantzigModel class' initialize() - # method to populate it - annot = "Dantzig's transportation problem for illustration and testing" - scen = Scenario( - mp, - **models["dantzig"], # type: ignore [arg-type] + # Initialize a new Scenario, and use the DantzigModel class' initialize() method to + # populate it + args: "ScenarioKwargs" = dict( + **models["dantzig"], version="new", - annotation=annot, + annotation="Dantzig's transportation problem for illustration and testing", scheme="dantzig", with_data=True, ) + # Use a distinct scenario name for a particular test + args.update({"scenario": request.node.name} if request else {}) + + scen = Scenario(mp, **args) - # commit the scenario + # Commit the scenario scen.commit("Import Dantzig's transport problem for testing.") - # set this new scenario as the default version for the model/scenario name + # Set this new scenario as the default version for the model/scenario name scen.set_as_default() if solve: # Solve the model using the GAMS code provided in the `tests` folder scen.solve(model="dantzig", case="transport_standard", quiet=quiet) - # add timeseries data for testing `clone(keep_solution=False)` - # and `remove_solution()` + # Add time series data for testing clone(keep_solution=False) and remove_solution() scen.check_out(timeseries_only=True) scen.add_timeseries(HIST_DF, meta=True) scen.add_timeseries(INP_DF) diff --git a/ixmp/testing/jupyter.py b/ixmp/testing/jupyter.py index 3c3cdbf9e..7850dc50b 100644 --- a/ixmp/testing/jupyter.py +++ b/ixmp/testing/jupyter.py @@ -48,16 +48,6 @@ def run_notebook(nb_path, tmp_path, env=None, **kwargs): import nbformat from nbclient import NotebookClient - # Workaround for https://github.com/jupyter/nbclient/issues/85 - if ( - sys.version_info[0] == 3 - and sys.version_info[1] >= 8 - and sys.platform.startswith("win") - ): - import asyncio - - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - # Read the notebook nb = nbformat.read(nb_path, as_version=4) diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index 4020d9066..b84920bea 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -135,14 +135,14 @@ def test_cache_invalidate(self, test_mp): backend.cache_invalidate(ts, "par", "baz", dict(x=["x1", "x2"], y=["y1", "y2"])) - def test_del_ts(self, test_mp): + def test_del_ts(self, test_mp, request): """Test CachingBackend.del_ts().""" # Since CachingBackend is an abstract class, test it via JDBCBackend - backend = test_mp._backend + backend: CachingBackend = test_mp._backend # type: ignore cache_size_pre = len(backend._cache) # Load data, thereby adding to the cache - s = make_dantzig(test_mp) + s = make_dantzig(test_mp, request=request) s.par("d") # Cache size has increased @@ -155,7 +155,7 @@ def test_del_ts(self, test_mp): s.__del__() # Force deletion of cached objects associated with `s` # Delete the object; associated cache is freed - del s + backend.del_ts(s) # Objects were invalidated/removed from cache assert cache_size_pre == len(backend._cache) diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index e59d4703e..aebfc56bb 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -2,7 +2,6 @@ import logging import os import platform -import sys from sys import getrefcount from typing import Tuple @@ -47,13 +46,7 @@ def test_jvm_warn(recwarn): assert len(recwarn) == 0, recwarn.pop().message -@pytest.mark.flaky( - reruns=5, - rerun_delay=2, - condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Windows", - reason="Flaky; see iiasa/ixmp#489", -) -def test_close(test_mp_f, capfd): +def test_close_default_logging(test_mp_f, capfd): """Platform.close_db() doesn't throw needless exceptions.""" # Use the session-scoped fixture to avoid affecting other tests in this file mp = test_mp_f @@ -68,9 +61,21 @@ def test_close(test_mp_f, capfd): captured = capfd.readouterr() assert captured.out == "" - # With log level INFO, a message is printed + +def test_close_increased_logging(test_mp_f, capfd): + """Platform.close_db() doesn't throw needless exceptions.""" + # Use the session-scoped fixture to avoid affecting other tests in this file + mp = test_mp_f + + # Close once + mp.close_db() + + # Set higher log level INFO level = mp.get_log_level() mp.set_log_level(logging.INFO) + + # Close again, once already closed + # With logging.INFO, a message is printed mp.close_db() captured = capfd.readouterr() msg = "Database connection could not be closed or was already closed" @@ -237,19 +242,16 @@ def test_invalid_properties_file(test_data_path): ixmp.Platform(dbprops=test_data_path / "hsqldb.properties") -@pytest.mark.flaky( - reruns=5, - rerun_delay=2, - condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Windows", - reason="Flaky; see iiasa/ixmp#489", -) -def test_connect_message(capfd, caplog): - msg = "connected to database 'jdbc:hsqldb:mem://ixmptest' (user: ixmp)..." +def test_connect_message(capfd, caplog, request): + msg = ( + f"connected to database 'jdbc:hsqldb:mem://{request.node.name}_0' " + "(user: ixmp)..." + ) ixmp.Platform( backend="jdbc", driver="hsqldb", - url="jdbc:hsqldb:mem://ixmptest", + url=f"jdbc:hsqldb:mem://{request.node.name}_0", log_level="INFO", ) @@ -260,11 +262,14 @@ def test_connect_message(capfd, caplog): # a previous run may have left the Java log level higher than INFO, in which # case the Java Platform object would not write to stderr before set_log_level() # in the above call. Try again now that the level is INFO: - + msg = ( + f"connected to database 'jdbc:hsqldb:mem://{request.node.name}_1' " + "(user: ixmp)..." + ) ixmp.Platform( backend="jdbc", driver="hsqldb", - url="jdbc:hsqldb:mem://ixmptest", + url=f"jdbc:hsqldb:mem://{request.node.name}_1", ) # Instead, log messages are printed to stdout @@ -273,7 +278,7 @@ def test_connect_message(capfd, caplog): @pytest.mark.parametrize("arg", [True, False]) -def test_cache_arg(arg): +def test_cache_arg(arg, request): """Test 'cache' argument, passed to CachingBackend.""" mp = ixmp.Platform( backend="jdbc", @@ -281,7 +286,7 @@ def test_cache_arg(arg): url="jdbc:hsqldb:mem://test_cache_false", cache=arg, ) - scen = make_dantzig(mp) + scen = make_dantzig(mp, request=request) # Maybe put something in the cache scen.par("a") @@ -339,8 +344,8 @@ def test_init(tmp_env, args, kwargs, action, kind, match): ixmp.Platform(*args, **kwargs) -def test_gh_216(test_mp): - scen = make_dantzig(test_mp) +def test_gh_216(test_mp, request): + scen = make_dantzig(test_mp, request=request) filters = dict(i=["seattle", "beijing"]) @@ -376,30 +381,32 @@ def test_verbose_exception(test_mp, exception_verbose_true): assert "at.ac.iiasa.ixmp.Platform.getScenario" in exc_msg -@pytest.mark.xfail( - condition=sys.version_info.minor <= 10, - raises=AssertionError, - # See also test_base.TestCachingBackend.test_del_ts - reason="https://github.com/iiasa/ixmp/issues/463", +@pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Windows", + reason="Flaky; see iiasa/ixmp#543", ) -def test_del_ts(): +def test_del_ts(request): mp = ixmp.Platform( backend="jdbc", driver="hsqldb", url="jdbc:hsqldb:mem:test_del_ts", ) + backend: ixmp.backend.jdbc.JDBCBackend = mp._backend # type: ignore + # Number of Java objects referenced by the JDBCBackend - N_obj = len(mp._backend.jindex) + N_obj = len(backend.jindex) # Create a list of some Scenario objects N = 8 - scenarios = [make_dantzig(mp)] + scenarios = [make_dantzig(mp, request=request)] for i in range(1, N): - scenarios.append(scenarios[0].clone(scenario=f"clone {i}")) + scenarios.append(scenarios[0].clone(scenario=f"{request.node.name} clone {i}")) # Number of referenced objects has increased by 8 - assert len(mp._backend.jindex) == N_obj + N + assert len(backend.jindex) == N_obj + N # Pop and free the objects for i in range(N): @@ -414,22 +421,23 @@ def test_del_ts(): s_id = id(s) # Underlying Java object - s_jobj = mp._backend.jindex[s] + s_jobj = backend.jindex[s] # Now delete the Scenario object - del s + # del s # should work, but doesn't always resolve to s.__del__() + backend.del_ts(s) # Number of referenced objects decreases by 1 - assert len(mp._backend.jindex) == N_obj + N - (i + 1) + assert len(backend.jindex) == N_obj + N - (i + 1) # ID is no longer in JDBCBackend.jindex - assert s_id not in mp._backend.jindex + assert s_id not in backend.jindex # s_jobj is the only remaining reference to the Java object assert getrefcount(s_jobj) - 1 == 1 del s_jobj # Backend is again empty - assert len(mp._backend.jindex) == N_obj + assert len(backend.jindex) == N_obj # NB coverage is omitted because this test is not included in the standard suite @@ -648,8 +656,8 @@ def test_reload_cycle( memory_usage("shutdown") -def test_docs(test_mp): - scen = make_dantzig(test_mp) +def test_docs(test_mp, request): + scen = make_dantzig(test_mp, request=request) # test model docs test_mp.set_doc("model", {scen.model: "Dantzig model"}) assert test_mp.get_doc("model") == {"canning problem": "Dantzig model"} @@ -672,9 +680,9 @@ def test_docs(test_mp): assert ex.value.args[0] == exp -def test_cache_clear(test_mp): +def test_cache_clear(test_mp, request): """Removing set elements causes the cache to be cleared entirely.""" - scen = make_dantzig(test_mp) + scen = make_dantzig(test_mp, request=request) # Load an item so that it is cached d0 = scen.par("d") diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index c61460a39..6259066fb 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -696,7 +696,7 @@ def test_filter_str(scen_empty): assert_frame_equal(exp[["s", "value"]], obs[["s", "value"]]) -def test_solve_callback(test_mp): +def test_solve_callback(test_mp, request): """Test the callback argument to Scenario.solve(). In real usage, callback() would compute some kind of convergence criterion. This @@ -705,7 +705,7 @@ def test_solve_callback(test_mp): equals an expected value, and the model has 'converged'. """ # Set up the Dantzig problem - scen = make_dantzig(test_mp) + scen = make_dantzig(test_mp, request=request) # Solve the scenario as configured solve_args = dict(model="dantzig", quiet=True) diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index 50a6a6288..f7af7263b 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -87,7 +87,7 @@ def ts(self, request, mp): node = hash(request.node.nodeid.replace("/", " ")) # Class of object to yield cls = request.param - yield cls(mp, model=f"test-{node}", scenario="test", version="new") + yield cls(mp, model=f"test-{node}", scenario=f"test-{node}", version="new") # Initialize TimeSeries @pytest.mark.parametrize("cls", [TimeSeries, Scenario]) diff --git a/ixmp/tests/data/access.properties b/ixmp/tests/data/test_check_multi_model_access.properties similarity index 100% rename from ixmp/tests/data/access.properties rename to ixmp/tests/data/test_check_multi_model_access.properties diff --git a/ixmp/tests/data/test_check_single_model_access.properties b/ixmp/tests/data/test_check_single_model_access.properties new file mode 100644 index 000000000..24a595ba8 --- /dev/null +++ b/ixmp/tests/data/test_check_single_model_access.properties @@ -0,0 +1,16 @@ +# Used by test_access.py + +config.name = unit_test_db@local + +jdbc.driver = org.hsqldb.jdbcDriver +jdbc.url = jdbc:hsqldb:mem:test_access +jdbc.user = ixmp +jdbc.pwd = ixmp + +application.tag = IXSE_SR15 +application.serverURL = http://localhost:8888 + +config.server.url = {auth_url} +config.server.config = DemoDB +config.server.username = service_user_dev +config.server.password = service_user_dev diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 9e653c3b8..04ddfc95f 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -27,8 +27,8 @@ pytestmark = pytest.mark.usefixtures("parametrize_quantity_class") -def test_from_url(test_mp) -> None: - ts = make_dantzig(test_mp) +def test_from_url(test_mp, request) -> None: + ts = make_dantzig(test_mp, request=request) full_url = f"ixmp://{ts.platform.name}/{ts.url}" @@ -45,8 +45,8 @@ def test_from_url(test_mp) -> None: assert ts.url == result.url -def test_get_remove_ts(caplog, test_mp) -> None: - ts = make_dantzig(test_mp) +def test_get_remove_ts(caplog, test_mp, request) -> None: + ts = make_dantzig(test_mp, request=request) caplog.set_level(logging.INFO, "ixmp") @@ -107,8 +107,8 @@ def test_map_as_qty() -> None: assert_qty_equal(exp, result) -def test_update_scenario(caplog, test_mp) -> None: - scen = make_dantzig(test_mp) +def test_update_scenario(caplog, test_mp, request) -> None: + scen = make_dantzig(test_mp, request=request) scen.check_out() scen.add_set("j", "toronto") scen.commit("Add j=toronto") diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index 230f9ad19..22e9cdf35 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -23,7 +23,7 @@ def scenario(test_mp): @pytest.mark.usefixtures("protect_rename_dims") -def test_configure(test_mp, test_data_path) -> None: +def test_configure(test_mp, test_data_path, request) -> None: # Configure globally; handles 'rename_dims' section configure(rename_dims={"i": "i_renamed"}) @@ -33,7 +33,7 @@ def test_configure(test_mp, test_data_path) -> None: assert "i" in RENAME_DIMS # Reporting uses the RENAME_DIMS mapping of 'i' to 'i_renamed' - scen = make_dantzig(test_mp) + scen = make_dantzig(test_mp, request=request) rep = Reporter.from_scenario(scen) assert "d:i_renamed-j" in rep, rep.graph.keys() assert ["seattle", "san-diego"] == rep.get("i_renamed") @@ -142,10 +142,10 @@ def test_platform_units(test_mp, caplog, ureg) -> None: assert unit.dimensionality == {"[USD]": 1, "[pkm]": -1} -def test_cli(ixmp_cli, test_mp, test_data_path) -> None: +def test_cli(ixmp_cli, test_mp, test_data_path, request) -> None: # Put something in the database test_mp.open_db() - make_dantzig(test_mp) + make_dantzig(test_mp, request=request) test_mp.close_db() platform_name = test_mp.name @@ -159,7 +159,7 @@ def test_cli(ixmp_cli, test_mp, test_data_path) -> None: "--model", "canning problem", "--scenario", - "standard", + f"{request.node.name}", "report", "--config", str(test_data_path / "report-config-0.yaml"), diff --git a/ixmp/tests/test_access.py b/ixmp/tests/test_access.py index ddf3330e6..42d6df993 100644 --- a/ixmp/tests/test_access.py +++ b/ixmp/tests/test_access.py @@ -1,6 +1,4 @@ import logging -import os -import platform import sys from subprocess import Popen from time import sleep @@ -52,13 +50,7 @@ def mock(server): yield httpmock -@pytest.mark.flaky( - reruns=5, - rerun_delay=2, - condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", - reason="Flaky; see iiasa/ixmp#489", -) -def test_check_single_model_access(mock, tmp_path, test_data_path): +def test_check_single_model_access(mock, tmp_path, test_data_path, request): mock.when( "POST /access/list", body='.+"test_user".+', @@ -71,7 +63,10 @@ def test_check_single_model_access(mock, tmp_path, test_data_path): ).reply("[false]", headers={"Content-Type": "application/json"}, times=FOREVER) test_props = create_test_platform( - tmp_path, test_data_path, "access", auth_url=mock.pretend_url + tmp_path, + test_data_path, + f"{request.node.name}", + auth_url=mock.pretend_url, ) mp = ixmp.Platform(backend="jdbc", dbprops=test_props) @@ -86,7 +81,7 @@ def test_check_single_model_access(mock, tmp_path, test_data_path): assert not granted -def test_check_multi_model_access(mock, tmp_path, test_data_path): +def test_check_multi_model_access(mock, tmp_path, test_data_path, request): mock.when( "POST /access/list", body='.+"test_user".+', @@ -103,7 +98,7 @@ def test_check_multi_model_access(mock, tmp_path, test_data_path): ) test_props = create_test_platform( - tmp_path, test_data_path, "access", auth_url=mock.pretend_url + tmp_path, test_data_path, f"{request.node.name}", auth_url=mock.pretend_url ) mp = ixmp.Platform(backend="jdbc", dbprops=test_props) diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 48dd48151..15c66db1d 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -165,8 +165,9 @@ def call(*args, exit_0=True): assert UsageError.exit_code == r.exit_code -def test_platform_copy(ixmp_cli, tmp_path): +def test_platform_copy(ixmp_cli, tmp_path, request): """Test 'platform' command.""" + test_specific_name = request.node.name def call(*args, exit_0=True): result = ixmp_cli.invoke(["platform"] + list(map(str, args))) @@ -174,30 +175,47 @@ def call(*args, exit_0=True): return result # Add some temporary platform configuration - call("add", "p1", "jdbc", "oracle", "HOSTNAME", "USER", "PASSWORD") - call("add", "p2", "jdbc", "hsqldb", tmp_path.joinpath("p2")) + call( + "add", + f"p1-{test_specific_name}", + "jdbc", + "oracle", + "HOSTNAME", + "USER", + "PASSWORD", + ) + call( + "add", + f"p2-{test_specific_name}", + "jdbc", + "hsqldb", + tmp_path.joinpath(f"p2-{test_specific_name}"), + ) # Force connection to p2 so that files are created - ixmp_cli.invoke(["--platform=p2", "list"]) + ixmp_cli.invoke([f"--platform=p2-{test_specific_name}", "list"]) # Dry-run produces expected output - r = call("copy", "p2", "p3") - assert re.search("Copy .*p2.script → .*p3.script", r.output) + r = call("copy", f"p2-{test_specific_name}", f"p3-{test_specific_name}") + assert re.search( + f"Copy .*p2-{test_specific_name}.script → .*p3-{test_specific_name}.script", + r.output, + ) with pytest.raises(ValueError): # New platform configuration is not saved - ixmp.config.get_platform_info("p3") + ixmp.config.get_platform_info(f"p3-{test_specific_name}") # --go actually copies files, saves new platform config - r = call("copy", "--go", "p2", "p3") - assert tmp_path.joinpath("p3.script").exists() - assert ixmp.config.get_platform_info("p3") + r = call("copy", "--go", f"p2-{test_specific_name}", f"p3-{test_specific_name}") + assert tmp_path.joinpath(f"p3-{test_specific_name}.script").exists() + assert ixmp.config.get_platform_info(f"p3-{test_specific_name}") # Dry-run again with existing config and files - r = call("copy", "p2", "p3") + r = call("copy", f"p2-{test_specific_name}", f"p3-{test_specific_name}") assert "would replace existing file" in r.output # Copying a non-HyperSQL-backed platform fails with pytest.raises(AssertionError): - call("copy", "p1", "p3") + call("copy", f"p1-{test_specific_name}", f"p3-{test_specific_name}") def test_import_ts(ixmp_cli, test_mp, test_data_path): diff --git a/ixmp/tests/test_integration.py b/ixmp/tests/test_integration.py index f5f3a2753..caf5d6b88 100644 --- a/ixmp/tests/test_integration.py +++ b/ixmp/tests/test_integration.py @@ -1,6 +1,4 @@ import logging -import os -import platform import numpy as np import pytest @@ -14,7 +12,7 @@ TS_DF_CLEARED.loc[0, 2005] = np.nan -def test_run_clone(caplog, test_mp): +def test_run_clone(caplog, test_mp, request): caplog.set_level(logging.WARNING) # this test is designed to cover the full functionality of the GAMS API @@ -24,14 +22,20 @@ def test_run_clone(caplog, test_mp): # - reads back the solution from the output # - performs the test on the objective value and the timeseries data mp = test_mp - scen = make_dantzig(mp, solve=True, quiet=True) + scen = make_dantzig(mp, solve=True, quiet=True, request=request) assert np.isclose(scen.var("z")["lvl"], 153.675) - assert_frame_equal(scen.timeseries(iamc=True), TS_DF) + assert_frame_equal( + scen.timeseries(iamc=True), + TS_DF.assign(scenario=[scen.scenario, scen.scenario]), + ) # cloning with `keep_solution=True` keeps all timeseries and the solution scen2 = scen.clone(keep_solution=True) assert np.isclose(scen2.var("z")["lvl"], 153.675) - assert_frame_equal(scen2.timeseries(iamc=True), TS_DF) + assert_frame_equal( + scen2.timeseries(iamc=True), + TS_DF.assign(scenario=[scen.scenario, scen.scenario]), + ) # version attribute of the clone increments the original (GitHub #211) assert scen2.version == scen.version + 1 @@ -47,20 +51,25 @@ def test_run_clone(caplog, test_mp): # timeseries set as `meta=True` scen3 = scen.clone(keep_solution=False) assert np.isnan(scen3.var("z")["lvl"]) - assert_frame_equal(scen3.timeseries(iamc=True), HIST_DF) + assert_frame_equal( + scen3.timeseries(iamc=True), HIST_DF.assign(scenario=scen.scenario) + ) # cloning with `keep_solution=False` and `first_model_year` # drops the solution and removes all timeseries not marked `meta=True` # in the model horizon (i.e, `year >= first_model_year`) scen4 = scen.clone(keep_solution=False, shift_first_model_year=2005) assert np.isnan(scen4.var("z")["lvl"]) - assert_frame_equal(scen4.timeseries(iamc=True), TS_DF_CLEARED) + assert_frame_equal( + scen4.timeseries(iamc=True), + TS_DF_CLEARED.assign(scenario=[scen.scenario, scen.scenario]), + ) -def test_run_remove_solution(test_mp): +def test_run_remove_solution(test_mp, request): # create a new instance of the transport problem and solve it mp = test_mp - scen = make_dantzig(mp, solve=True, quiet=True) + scen = make_dantzig(mp, solve=True, quiet=True, request=request) assert np.isclose(scen.var("z")["lvl"], 153.675) # check that re-solving the model will raise an error if a solution exists @@ -72,7 +81,9 @@ def test_run_remove_solution(test_mp): scen2.remove_solution() assert not scen2.has_solution() assert np.isnan(scen2.var("z")["lvl"]) - assert_frame_equal(scen2.timeseries(iamc=True), HIST_DF) + assert_frame_equal( + scen2.timeseries(iamc=True), HIST_DF.assign(scenario=scen.scenario) + ) # remove the solution with a specific year as first model year, check that # variables are empty and timeseries not marked `meta=True` are removed @@ -80,7 +91,10 @@ def test_run_remove_solution(test_mp): scen3.remove_solution(first_model_year=2005) assert not scen3.has_solution() assert np.isnan(scen3.var("z")["lvl"]) - assert_frame_equal(scen3.timeseries(iamc=True), TS_DF_CLEARED) + assert_frame_equal( + scen3.timeseries(iamc=True), + TS_DF_CLEARED.assign(scenario=[scen.scenario, scen.scenario]), + ) def scenario_list(mp): @@ -95,16 +109,10 @@ def get_distance(scen): return scen.par("d").set_index(["i", "j"]).loc["san-diego", "topeka"]["value"] -@pytest.mark.flaky( - reruns=5, - rerun_delay=2, - condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", - reason="Flaky; see iiasa/ixmp#489", -) -def test_multi_db_run(tmpdir): +def test_multi_db_run(tmpdir, request): # create a new instance of the transport problem and solve it mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1") - scen1 = make_dantzig(mp1, solve=True, quiet=True) + scen1 = make_dantzig(mp1, solve=True, quiet=True, request=request) mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") # add other unit to make sure that the mapping is correct during clone @@ -124,7 +132,9 @@ def test_multi_db_run(tmpdir): # reopen the connection to the second platform and reload scenario _mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") assert_multi_db(mp1, _mp2) - scen2 = ixmp.Scenario(_mp2, **models["dantzig"]) + args = models["dantzig"].copy() + args.update(scenario=request.node.name) + scen2 = ixmp.Scenario(_mp2, **args) # check that sets, variables and parameter were copied correctly assert_array_equal(scen1.set("i"), scen2.set("i")) @@ -135,13 +145,16 @@ def test_multi_db_run(tmpdir): # check that custom unit, region and timeseries are migrated correctly assert scen2.par("f")["value"] == 90.0 assert scen2.par("f")["unit"] == "USD/km" - assert_frame_equal(scen2.timeseries(iamc=True), TS_DF) + assert_frame_equal( + scen2.timeseries(iamc=True), + TS_DF.assign(scenario=[scen2.scenario, scen2.scenario]), + ) -def test_multi_db_edit_source(tmpdir): +def test_multi_db_edit_source(tmpdir, request): # create a new instance of the transport problem mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1") - scen1 = make_dantzig(mp1) + scen1 = make_dantzig(mp1, request=request) mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") scen2 = scen1.clone(platform=mp2) @@ -163,10 +176,10 @@ def test_multi_db_edit_source(tmpdir): assert_multi_db(mp1, mp2) -def test_multi_db_edit_target(tmpdir): +def test_multi_db_edit_target(tmpdir, request): # create a new instance of the transport problem mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1") - scen1 = make_dantzig(mp1) + scen1 = make_dantzig(mp1, request=request) mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") scen2 = scen1.clone(platform=mp2) diff --git a/ixmp/tests/test_model.py b/ixmp/tests/test_model.py index d9fd14ba2..83259563d 100644 --- a/ixmp/tests/test_model.py +++ b/ixmp/tests/test_model.py @@ -23,9 +23,9 @@ class M1(Model): M1() -def test_model_initialize(test_mp, caplog): +def test_model_initialize(test_mp, caplog, request): # Model.initialize runs on an empty Scenario - s = make_dantzig(test_mp) + s = make_dantzig(test_mp, request=request) b1 = s.par("b") assert len(b1) == 3 @@ -108,8 +108,8 @@ def test_gams_version(): class TestGAMSModel: @pytest.fixture(scope="class") - def dantzig(self, test_mp): - yield make_dantzig(test_mp) + def dantzig(self, test_mp, request): + yield make_dantzig(test_mp, request=request) @pytest.mark.parametrize("char", r'<>"/\|?*') def test_filename_invalid_char(self, dantzig, char): diff --git a/ixmp/tests/test_tutorials.py b/ixmp/tests/test_tutorials.py index 08fb739cd..6d82dc395 100644 --- a/ixmp/tests/test_tutorials.py +++ b/ixmp/tests/test_tutorials.py @@ -7,28 +7,41 @@ from ixmp.testing import get_cell_output, run_notebook +group_base_name = platform.system() + platform.python_version() + +GHA = "GITHUB_ACTIONS" in os.environ + + FLAKY = pytest.mark.flaky( reruns=5, rerun_delay=2, - condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", - reason="Flaky; see iiasa/ixmp#489", + condition=GHA and platform.system() == "Windows", + reason="Flaky; see iiasa/ixmp#543", ) +@pytest.fixture(scope="session") +def default_args(): + """Default arguments for :func:`.run_notebook.""" + # Use a longer timeout for GHA + return dict(timeout=30) if GHA else dict() + + @FLAKY -def test_py_transport(tutorial_path, tmp_path, tmp_env): +@pytest.mark.xdist_group(name=f"{group_base_name}-0") +def test_py_transport(tutorial_path, tmp_path, tmp_env, default_args): fname = tutorial_path / "transport" / "py_transport.ipynb" - nb, errors = run_notebook(fname, tmp_path, tmp_env) + nb, errors = run_notebook(fname, tmp_path, tmp_env, **default_args) assert errors == [] # FIXME use get_cell_by_name instead of assuming cell count/order is fixed assert np.isclose(get_cell_output(nb, -5)["lvl"], 153.6750030517578) -@FLAKY -def test_py_transport_scenario(tutorial_path, tmp_path, tmp_env): +@pytest.mark.xdist_group(name=f"{group_base_name}-0") +def test_py_transport_scenario(tutorial_path, tmp_path, tmp_env, default_args): fname = tutorial_path / "transport" / "py_transport_scenario.ipynb" - nb, errors = run_notebook(fname, tmp_path, tmp_env) + nb, errors = run_notebook(fname, tmp_path, tmp_env, **default_args) assert errors == [] assert np.isclose(get_cell_output(nb, "scen-z")["lvl"], 153.675) @@ -36,24 +49,25 @@ def test_py_transport_scenario(tutorial_path, tmp_path, tmp_env): @FLAKY +@pytest.mark.xdist_group(name=f"{group_base_name}-1") @pytest.mark.rixmp # TODO investigate and resolve the cause of the time outs; remove this mark -@pytest.mark.skipif( - "GITHUB_ACTIONS" in os.environ and sys.platform == "linux", reason="Times out" -) -def test_R_transport(tutorial_path, tmp_path, tmp_env): +@pytest.mark.skipif(GHA and sys.platform == "linux", reason="Times out") +def test_R_transport(tutorial_path, tmp_path, tmp_env, default_args): fname = tutorial_path / "transport" / "R_transport.ipynb" - nb, errors = run_notebook(fname, tmp_path, tmp_env, kernel_name="IR") + nb, errors = run_notebook( + fname, tmp_path, tmp_env, kernel_name="IR", **default_args + ) assert errors == [] -@FLAKY +@pytest.mark.xdist_group(name=f"{group_base_name}-1") @pytest.mark.rixmp # TODO investigate and resolve the cause of the time outs; remove this mark -@pytest.mark.skipif( - "GITHUB_ACTIONS" in os.environ and sys.platform == "linux", reason="Times out" -) -def test_R_transport_scenario(tutorial_path, tmp_path, tmp_env): +@pytest.mark.skipif(GHA and sys.platform == "linux", reason="Times out") +def test_R_transport_scenario(tutorial_path, tmp_path, tmp_env, default_args): fname = tutorial_path / "transport" / "R_transport_scenario.ipynb" - nb, errors = run_notebook(fname, tmp_path, tmp_env, kernel_name="IR") + nb, errors = run_notebook( + fname, tmp_path, tmp_env, kernel_name="IR", **default_args + ) assert errors == [] diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 8cf70219a..05f12e37e 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -56,10 +56,10 @@ def test_check_year(): assert util.check_year(y3, s3) is True -def test_diff_identical(test_mp): +def test_diff_identical(test_mp, request): """diff() of identical Scenarios.""" - scen_a = make_dantzig(test_mp) - scen_b = make_dantzig(test_mp) + scen_a = make_dantzig(test_mp, request=request) + scen_b = make_dantzig(test_mp, request=request) # Compare identical scenarios: produces data of same length for name, df in util.diff(scen_a, scen_b): @@ -72,10 +72,10 @@ def test_diff_identical(test_mp): assert exp_name == name and len(df) == N -def test_diff_data(test_mp): +def test_diff_data(test_mp, request): """diff() when Scenarios contain the same items, but different data.""" - scen_a = make_dantzig(test_mp) - scen_b = make_dantzig(test_mp) + scen_a = make_dantzig(test_mp, request=request) + scen_b = make_dantzig(test_mp, request=request) # Modify `scen_a` and `scen_b` scen_a.check_out() @@ -133,10 +133,10 @@ def test_diff_data(test_mp): pdt.assert_frame_equal(exp_d.iloc[[0, 3], :].reset_index(drop=True), df) -def test_diff_items(test_mp): +def test_diff_items(test_mp, request): """diff() when Scenarios contain the different items.""" - scen_a = make_dantzig(test_mp) - scen_b = make_dantzig(test_mp) + scen_a = make_dantzig(test_mp, request=request) + scen_b = make_dantzig(test_mp, request=request) # Modify `scen_a` and `scen_b` scen_a.check_out() @@ -156,11 +156,11 @@ def test_diff_items(test_mp): pass # No check of the contents -def test_discard_on_error(caplog, test_mp): +def test_discard_on_error(caplog, test_mp, request): caplog.set_level(logging.INFO, "ixmp.util") # Create a test scenario, checked-in state - s = make_dantzig(test_mp) + s = make_dantzig(test_mp, request=request) url = s.url # Some actions that don't trigger exceptions diff --git a/pyproject.toml b/pyproject.toml index 421a53896..54e1b867e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,7 @@ markers = [ "rixmp: test of the ixmp R interface.", "performance: ixmp performance test.", ] +tmp_path_retention_policy = "none" [tool.ruff.lint] select = ["C9", "E", "F", "I", "W"] diff --git a/tutorial/transport/R_transport_scenario.ipynb b/tutorial/transport/R_transport_scenario.ipynb index 44885e83f..3346c89ec 100644 --- a/tutorial/transport/R_transport_scenario.ipynb +++ b/tutorial/transport/R_transport_scenario.ipynb @@ -42,7 +42,7 @@ "library(reticulate)\n", "\n", "# Import ixmp and message_ix, just as in Python\n", - "ixmp <- import(\"ixmp\")" + "ixmp <- import(\"ixmp\")\n" ] }, { @@ -54,7 +54,7 @@ "outputs": [], "source": [ "# launch the ix modeling platform using the local default database\n", - "mp <- ixmp$Platform()" + "mp <- ixmp$Platform()\n" ] }, { @@ -66,7 +66,7 @@ "scen_list <- mp$scenario_list()\n", "scen_list\n", "\n", - "# TODO: the conversion of the Java output of the `scenario_list()` function to a clean R dataframe is not yet implemented" + "# TODO: the conversion of the Java output of the `scenario_list()` function to a clean R dataframe is not yet implemented\n" ] }, { @@ -79,7 +79,15 @@ "source": [ "# details for loading an existing datastructure from the IX modeling platform\n", "model <- \"transport problem\"\n", - "scenario <- \"standard\"" + "scenario <- \"standard\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you have just run the first, ``R_transport`` tutorial, the existing scenario should appear, and we can load it.\n", + "Uncomment and run the following line." ] }, { @@ -91,7 +99,24 @@ "outputs": [], "source": [ "# load the default version scenario from the first tutorial\n", - "scen <- ixmp$Scenario(mp, model, scenario)" + "# scen <- ixmp$Scenario(mp, model, scenario)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If not (e.g. starting with this tutorial), we can use a function that creates the scenario from scratch in one step:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ixmp_testing <- import(\"ixmp.testing\")\n", + "scen <- ixmp_testing$make_dantzig(mp, solve = \".\")\n" ] }, { @@ -110,8 +135,8 @@ "outputs": [], "source": [ "# load the distance parameter\n", - "d = scen$par(\"d\")\n", - "d" + "d <- scen$par(\"d\")\n", + "d\n" ] }, { @@ -121,7 +146,7 @@ "outputs": [], "source": [ "# show only the distances for connections from Seattle\n", - "d[d['i'] == \"seattle\",]" + "d[d[\"i\"] == \"seattle\", ]\n" ] }, { @@ -132,12 +157,12 @@ "source": [ "# for faster access or more complex filtering,\n", "# it may be easier to only load specific parameter elements using a dictionary\n", - "ele_filter = {}\n", - "ele_filter$i = c('seattle')\n", - "ele_filter$j = c('chicago', 'topeka')\n", + "ele_filter <- {}\n", + "ele_filter$i <- c(\"seattle\")\n", + "ele_filter$j <- c(\"chicago\", \"topeka\")\n", "\n", - "d_filtered = scen$par(\"d\", ele_filter)\n", - "d_filtered" + "d_filtered <- scen$par(\"d\", ele_filter)\n", + "d_filtered\n" ] }, { @@ -159,7 +184,7 @@ "outputs": [], "source": [ "# create a new scenario by cloning the datastructure (without keeping the solution)\n", - "scen_detroit <- scen$clone(model, 'detroit', annotation='extend the Transport problem by a new city', keep_solution=FALSE)" + "scen_detroit <- scen$clone(model, \"detroit\", annotation = \"extend the Transport problem by a new city\", keep_solution = FALSE)\n" ] }, { @@ -171,7 +196,7 @@ "outputs": [], "source": [ "# check out the datastructure to make changes\n", - "scen_detroit$check_out()" + "scen_detroit$check_out()\n" ] }, { @@ -180,15 +205,15 @@ "metadata": {}, "outputs": [], "source": [ - "# reduce demand \n", - "scen_detroit$add_par('b', 'chicago', 200, 'cases')\n", + "# reduce demand\n", + "scen_detroit$add_par(\"b\", \"chicago\", 200, \"cases\")\n", "\n", "# add a new city with demand and distances\n", - "scen_detroit$add_set('j', 'detroit')\n", - "scen_detroit$add_par('b', 'detroit', 150, 'cases')\n", + "scen_detroit$add_set(\"j\", \"detroit\")\n", + "scen_detroit$add_par(\"b\", \"detroit\", 150, \"cases\")\n", "\n", - "d_add = data.frame(i = c('seattle','san-diego'), j = c('detroit','detroit'), value = c(1.7,1.9) , unit = 'cases')\n", - "scen_detroit$add_par('d', d_add)" + "d_add <- data.frame(i = c(\"seattle\", \"san-diego\"), j = c(\"detroit\", \"detroit\"), value = c(1.7, 1.9), unit = \"cases\")\n", + "scen_detroit$add_par(\"d\", d_add)\n" ] }, { @@ -199,8 +224,8 @@ }, "outputs": [], "source": [ - "d_add = data.frame(i = c('seattle','san-diego'), j = c('detroit','detroit'), value = c(1.7,1.9) , unit = 'cases')\n", - "scen_detroit$add_par('d', d_add)" + "d_add <- data.frame(i = c(\"seattle\", \"san-diego\"), j = c(\"detroit\", \"detroit\"), value = c(1.7, 1.9), unit = \"cases\")\n", + "scen_detroit$add_par(\"d\", d_add)\n" ] }, { @@ -212,9 +237,9 @@ "outputs": [], "source": [ "# save changes to database\n", - "comment = \"add new city 'detroit' with demand, reduce demand in 'chicago'\"\n", - "scen_detroit$commit(comment) \n", - "scen_detroit$set_as_default()" + "comment <- \"add new city 'detroit' with demand, reduce demand in 'chicago'\"\n", + "scen_detroit$commit(comment)\n", + "scen_detroit$set_as_default()\n" ] }, { @@ -232,7 +257,7 @@ }, "outputs": [], "source": [ - "scen_detroit$solve(model='dantzig')" + "scen_detroit$solve(model = \"dantzig\")\n" ] }, { @@ -251,7 +276,7 @@ "outputs": [], "source": [ "# display the objective value of the solution in the baseline scenario\n", - "scen$var(\"z\")" + "scen$var(\"z\")\n" ] }, { @@ -261,7 +286,7 @@ "outputs": [], "source": [ "# display the objective value of the solution in the \"detroit\" scenario\n", - "scen_detroit$var(\"z\")" + "scen_detroit$var(\"z\")\n" ] }, { @@ -271,7 +296,7 @@ "outputs": [], "source": [ "# display the quantities transported from canning plants to demand locations in the baseline scenario\n", - "scen$var(\"x\")" + "scen$var(\"x\")\n" ] }, { @@ -281,7 +306,7 @@ "outputs": [], "source": [ "# display the quantities transported from canning plants to demand locations in the \"detroit\" scenario\n", - "scen_detroit$var(\"x\")" + "scen_detroit$var(\"x\")\n" ] }, { @@ -291,7 +316,7 @@ "outputs": [], "source": [ "# display the quantities and marginals (=shadow prices) of the demand balance constraints in the baseline scenario\n", - "scen$equ(\"demand\")" + "scen$equ(\"demand\")\n" ] }, { @@ -301,7 +326,7 @@ "outputs": [], "source": [ "# display the quantities and marginals (=shadow prices) of the demand balance constraints in the \"detroit\" scenario\n", - "scen_detroit$equ(\"demand\")" + "scen_detroit$equ(\"demand\")\n" ] }, { @@ -320,7 +345,7 @@ "outputs": [], "source": [ "# close the connection of the platform instance to the local ixmp database files\n", - "mp$close_db()" + "mp$close_db()\n" ] }, {