Skip to content

Commit

Permalink
Fix: MLPRegressor tests (#1367)
Browse files Browse the repository at this point in the history
* Fix: MLPRegressor tests

* Fix: Ordering of statements in test

* Fix: MLP n_calls
  • Loading branch information
eddiebergman committed Jan 25, 2022
1 parent 6a6f6f1 commit bb08d04
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 35 deletions.
59 changes: 31 additions & 28 deletions test/test_pipeline/components/regression/test_base.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Type
from typing import Type, Container

import unittest

Expand Down Expand Up @@ -41,31 +41,29 @@ def test_default_boston(self):
Regressor=self.module
)

score = sklearn.metrics.r2_score(y_true=targets, y_pred=predictions)

# Special treatment for Gaussian Process Regression
if "default_boston_le_ge" in self.res:
# Special treatment for Gaussian Process Regression
self.assertLessEqual(
sklearn.metrics.r2_score(y_true=targets, y_pred=predictions),
self.res["default_boston_le_ge"][0]
)
self.assertGreaterEqual(
sklearn.metrics.r2_score(y_true=targets, y_pred=predictions),
self.res["default_boston_le_ge"][1]
)
upper, lower = self.res["default_boston_le_ge"]
assert lower <= score <= upper

else:
score = sklearn.metrics.r2_score(targets, predictions)
fixture = self.res["default_boston"]
places = self.res.get("default_boston_places", 7)

if score < -1e10:
print(f"score = {score}, fixture = {fixture}")
score = np.log(-score)
fixture = np.log(-fixture)
self.assertAlmostEqual(
fixture,
score,
places=self.res.get("default_boston_places", 7),
)

if self.res.get("boston_n_calls"):
self.assertEqual(self.res["boston_n_calls"], n_calls)
self.assertAlmostEqual(fixture, score, places)

if "boston_n_calls" in self.res:
expected = self.res["boston_n_calls"]
if isinstance(expected, Container):
assert n_calls in expected
else:
assert n_calls == expected

def test_default_boston_iterative_fit(self):

Expand All @@ -84,23 +82,28 @@ def test_default_boston_iterative_fit(self):

score = sklearn.metrics.r2_score(targets, predictions)
fixture = self.res["default_boston_iterative"]
places = self.res.get("default_boston_iterative_places", 7)

if score < -1e10:
print(f"score = {score}, fixture = {fixture}")
score = np.log(-score)
fixture = np.log(-fixture)

self.assertAlmostEqual(
fixture,
score,
places=self.res.get("default_boston_iterative_places", 7),
)
self.assertAlmostEqual(fixture, score, places)

if self.step_hyperparameter is not None:
self.assertEqual(
getattr(regressor.estimator, self.step_hyperparameter['name']),
self.res.get("boston_iterative_n_iter", self.step_hyperparameter['value'])
)
param_name = self.step_hyperparameter['name']
default = self.step_hyperparameter['value']

value = getattr(regressor.estimator, param_name)
expected = self.res.get("boston_iterative_n_iter", default)

# To currently allow for MLPRegressor which is indeterministic,
# we can have multiple values
if isinstance(expected, Container):
assert value in expected
else:
assert value == expected

def test_default_boston_iterative_sparse_fit(self):

Expand Down
31 changes: 24 additions & 7 deletions test/test_pipeline/components/regression/test_mlp.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from typing import Any, Dict

import sklearn.neural_network

from autosklearn.pipeline.components.regression.mlp import MLPRegressor
Expand All @@ -22,21 +24,36 @@ class MLPComponentTest(BaseRegressionComponentTest):
#
# These seem to have consistent CPU's so I'm unsure what the underlying reason
# for this to randomly fail only sometimes on Github runners
#
# Edit: If changing, please tracke what values were failing
#
# Seems there is a consistently different values for boston so:
# * include two valuess for n_iter in 'boston_iterative_n_iter'
# known-values = [236, 331]
#
# * decreased places from 6 -> 5 in 'default_boston_{sparse,_iterative_sparse}'
# to check for for iterations and expanded the default places for checking
# know-values = [-0.10972947168054104, -0.10973142976866268]
#
# * decreased places from 3 -> 1 in 'default_boston_places'
# known-values = [0.29521793994422807, 0.2750079862455884]
#
# * Include two value for 'boston_n_calls'
# known-values = [8, 9]
__test__ = True

__test__ = True

res = dict()
res: Dict[str, Any] = {}
res["default_boston"] = 0.2750079862455884
res["default_boston_places"] = 3
res["boston_n_calls"] = 8
res["boston_iterative_n_iter"] = 236
res["default_boston_places"] = 1
res["boston_n_calls"] = [8, 9]
res["boston_iterative_n_iter"] = [236, 331]
res["default_boston_iterative"] = res["default_boston"]
res["default_boston_iterative_places"] = 1
res["default_boston_sparse"] = -0.10972947168054104
res["default_boston_sparse_places"] = 6
res["default_boston_sparse_places"] = 5
res["default_boston_iterative_sparse"] = res["default_boston_sparse"]
res["default_boston_iterative_sparse_places"] = 6
res["default_boston_iterative_sparse_places"] = res["default_boston_sparse_places"]
res["default_diabetes"] = 0.35917389841850555
res["diabetes_n_calls"] = 9
res["diabetes_iterative_n_iter"] = 435
Expand Down

0 comments on commit bb08d04

Please sign in to comment.