From 69f1fc849327ab3c4a3ac7b0394c44b05077fbaf Mon Sep 17 00:00:00 2001 From: Valentin Laurent Date: Thu, 6 Feb 2025 17:17:36 +0100 Subject: [PATCH] TEST: create a single folder for all v1 tests, add test to compare prefit/not prefit, remove it from v0 tests, add non-integration tests to CI --- .gitignore | 2 +- Makefile | 37 +++++++++--- mapie/tests/test_regression.py | 27 --------- tests_v1/integration_tests/tests/pytest.ini | 3 + .../tests/test_regression.py | 4 +- .../integration_tests/utils.py | 0 tests_v1/tests/test_regression.py | 56 +++++++++++++++++++ 7 files changed, 91 insertions(+), 38 deletions(-) create mode 100644 tests_v1/integration_tests/tests/pytest.ini rename {mapie_v1 => tests_v1}/integration_tests/tests/test_regression.py (99%) rename {mapie_v1 => tests_v1}/integration_tests/utils.py (100%) create mode 100644 tests_v1/tests/test_regression.py diff --git a/.gitignore b/.gitignore index 57070a8cf..ed7caa966 100644 --- a/.gitignore +++ b/.gitignore @@ -59,7 +59,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ -mapie_v1/integration_tests/mapie_v0_package +tests_v1/integration_tests/mapie_v0_package # Translations *.mo diff --git a/Makefile b/Makefile index 0afef1f0c..19eee67a7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: tests doc build +integration_tests_folder_name = tests_v1/integration_tests mapie_v0_folder_name = mapie_v0_package lint: @@ -13,19 +14,28 @@ v1-type-check: tests: pytest -vs --doctest-modules mapie - pytest -vs --doctest-modules mapie_v1 --ignore=mapie_v1/integration_tests + $(MAKE) v1-tests + $(MAKE) v1-docstring-tests -integration-tests-v1: - @pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./mapie_v1/integration_tests/$(mapie_v0_folder_name) >/dev/null 2>&1 - @mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0 - @- export PYTHONPATH="${PYTHONPATH}:./mapie_v1/integration_tests/$(mapie_v0_folder_name)"; pytest -vs mapie_v1/integration_tests/tests -k $(pattern) - @mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0 ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie +v1-tests: + pytest -vs tests_v1 --ignore=$(integration_tests_folder_name) -checks-v1-not-in-ci: - $(MAKE) v1-type-check - $(MAKE) integration-tests-v1 pattern=test +v1-docstring-tests: + pytest -vs --doctest-modules mapie_v1 --ignore=$(integration_tests_folder_name) + +v1-integration-tests: + @pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./$(integration_tests_folder_name)/$(mapie_v0_folder_name) >/dev/null 2>&1 + @mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0 + @- export PYTHONPATH="${PYTHONPATH}:./$(integration_tests_folder_name)/$(mapie_v0_folder_name)"; pytest -vs $(integration_tests_folder_name)/tests $(params) + @mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0 ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie + +v1-checks-not-in-ci: + $(MAKE) v1-type-check # Issues when trying to include it in CI, see task "Dépendances v1 + MyPy stricter in CI" in project board + $(MAKE) v1-integration-tests # We don't want to include this in CI, will be removed at v1 release + $(MAKE) v1-coverage # To add in CI when we reach 100% coverage: + # We may need to add the v1 test suite here if we remove some v0 tests, to keep a 100% coverage pytest -vsx \ --cov-branch \ --cov=mapie \ @@ -35,6 +45,15 @@ coverage: --cov-config=.coveragerc \ --no-cov-on-fail +v1-coverage: + pytest -vsx \ + --cov-branch \ + --cov=mapie_v1 \ + --cov-report term-missing \ + --pyargs tests_v1 --ignore=$(integration_tests_folder_name) \ + --cov-fail-under=100 \ + --no-cov-on-fail + doc: $(MAKE) html -C doc diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py index f19c30219..bcf34c7e4 100644 --- a/mapie/tests/test_regression.py +++ b/mapie/tests/test_regression.py @@ -417,33 +417,6 @@ def test_calibration_data_size_asymmetric_score(delta: float) -> None: mapie_reg.predict(Xt, alpha=1-delta) -def test_same_results_prefit_split() -> None: - """ - Test checking that if split and prefit method have exactly - the same data split, then we have exactly the same results. - """ - X, y = make_regression( - n_samples=500, n_features=10, noise=1.0, random_state=1 - ) - cv = ShuffleSplit(n_splits=1, test_size=0.1, random_state=random_state) - train_index, val_index = list(cv.split(X))[0] - X_train, X_calib = X[train_index], X[val_index] - y_train, y_calib = y[train_index], y[val_index] - - mapie_reg = MapieRegressor(method='base', cv=cv) - mapie_reg.fit(X, y) - y_pred_1, y_pis_1 = mapie_reg.predict(X, alpha=0.1) - - model = LinearRegression().fit(X_train, y_train) - mapie_reg = MapieRegressor(estimator=model, method='base', cv="prefit") - mapie_reg.fit(X_calib, y_calib) - y_pred_2, y_pis_2 = mapie_reg.predict(X, alpha=0.1) - - np.testing.assert_allclose(y_pred_1, y_pred_2) - np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0]) - np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0]) - - @pytest.mark.parametrize("strategy", [*STRATEGIES]) def test_results_for_same_alpha(strategy: str) -> None: """ diff --git a/tests_v1/integration_tests/tests/pytest.ini b/tests_v1/integration_tests/tests/pytest.ini new file mode 100644 index 000000000..6fcf120ce --- /dev/null +++ b/tests_v1/integration_tests/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + regression \ No newline at end of file diff --git a/mapie_v1/integration_tests/tests/test_regression.py b/tests_v1/integration_tests/tests/test_regression.py similarity index 99% rename from mapie_v1/integration_tests/tests/test_regression.py rename to tests_v1/integration_tests/tests/test_regression.py index 8dea54ad1..ec23315fc 100644 --- a/mapie_v1/integration_tests/tests/test_regression.py +++ b/tests_v1/integration_tests/tests/test_regression.py @@ -22,10 +22,12 @@ from mapiev0.regression import MapieRegressor as MapieRegressorV0 # noqa from mapiev0.regression import MapieQuantileRegressor as MapieQuantileRegressorV0 # noqa -from mapie_v1.integration_tests.utils import (filter_params, +from tests_v1.integration_tests.utils import (filter_params, train_test_split_shuffle) from sklearn.model_selection import LeaveOneOut, GroupKFold +pytestmark = pytest.mark.regression + RANDOM_STATE = 1 K_FOLDS = 3 N_BOOTSTRAPS = 30 diff --git a/mapie_v1/integration_tests/utils.py b/tests_v1/integration_tests/utils.py similarity index 100% rename from mapie_v1/integration_tests/utils.py rename to tests_v1/integration_tests/utils.py diff --git a/tests_v1/tests/test_regression.py b/tests_v1/tests/test_regression.py new file mode 100644 index 000000000..0dcdfd69c --- /dev/null +++ b/tests_v1/tests/test_regression.py @@ -0,0 +1,56 @@ +import numpy as np +import pytest +from sklearn.datasets import make_regression +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import train_test_split +from mapie_v1.regression import SplitConformalRegressor + +RANDOM_STATE = 1 + + +@pytest.fixture(scope="module") +def dataset(): + X, y = make_regression( + n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE + ) + X_train, X_conf_test, y_train, y_conf_test = train_test_split( + X, y, random_state=RANDOM_STATE + ) + X_conformalize, X_test, y_conformalize, y_test = train_test_split( + X_conf_test, y_conf_test, random_state=RANDOM_STATE + ) + return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test + + +@pytest.fixture +def predictions_scr_prefit(dataset): + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + regressor = LinearRegression() + regressor.fit(X_train, y_train) + scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True) + scr_prefit.conformalize(X_conformalize, y_conformalize) + return scr_prefit.predict_interval(X_test) + + +@pytest.fixture +def predictions_scr_not_prefit(dataset): + X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset + scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False) + scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize) + return scr_not_prefit.predict_interval(X_test) + + +def test_scr_same_intervals_prefit_not_prefit( + predictions_scr_prefit, predictions_scr_not_prefit +) -> None: + intervals_scr_prefit = predictions_scr_prefit[1] + intervals_scr_not_prefit = predictions_scr_not_prefit[1] + np.testing.assert_equal(intervals_scr_prefit, intervals_scr_not_prefit) + + +def test_scr_same_predictions_prefit_not_prefit( + predictions_scr_prefit, predictions_scr_not_prefit +) -> None: + predictions_scr_prefit = predictions_scr_prefit[0] + predictions_scr_not_prefit = predictions_scr_not_prefit[0] + np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit)