Refactor SVD features preprocessing step #768
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| * Introduces AddSVDFeaturesStep as a dedicated preprocessing step for SVD feature generation | ||
| * Removes SVD-related functionality from ReshapeFeatureDistributionsStep | ||
| * Extracts utility functions to a new `tabpfn/preprocessing/steps/utils.py` module |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Removes "scaler" as an option for `global_transformer_name` in `PreprocessorConfig` |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| """Adds SVD features to the data.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from typing import TYPE_CHECKING, Literal | ||
| from typing_extensions import override | ||
|
|
||
| from sklearn.decomposition import TruncatedSVD | ||
| from sklearn.pipeline import Pipeline | ||
| from sklearn.preprocessing import StandardScaler | ||
|
|
||
| from tabpfn.preprocessing.datamodel import FeatureModality, FeatureSchema | ||
| from tabpfn.preprocessing.pipeline_interface import PreprocessingStep | ||
| from tabpfn.preprocessing.steps.utils import make_standard_scaler_safe | ||
| from tabpfn.utils import infer_random_state | ||
|
|
||
| if TYPE_CHECKING: | ||
| import numpy as np | ||
|
|
||
|
|
||
| class AddSVDFeaturesStep(PreprocessingStep): | ||
| """Adds SVD features to the data.""" | ||
|
Contributor
There was a problem hiding this comment. @bejaeger can we elaborate slightly more the docstrings? What we do here is add, on top of the raw X, also more features that are just a compressed version of them? If so - I get it for numerical features, but a bit weird for categorical, and non-applicable for text? |
||
|
|
||
| def __init__( | ||
| self, | ||
| global_transformer_name: Literal[ | ||
| "svd", "svd_quarter_components" | ||
| ] = "svd_quarter_components", | ||
| random_state: int | np.random.Generator | None = None, | ||
| ): | ||
| """Initializes the AddSVDFeaturesStep.""" | ||
| super().__init__() | ||
| self.global_transformer_name = global_transformer_name | ||
| self.random_state = random_state | ||
| self.is_no_op: bool = False | ||
|
|
||
| def num_added_features(self, n_samples: int, n_features: int) -> int: | ||
| """Return the number of added features.""" | ||
| if n_features < 2: | ||
| return 0 | ||
|
|
||
| transformer = get_svd_features_transformer( | ||
| self.global_transformer_name, | ||
| n_samples, | ||
| n_features, | ||
| ) | ||
| return next( | ||
|
Contributor
There was a problem hiding this comment. @bejaeger I find this syntax hard to read. Why do we have next here? I'm confused |
||
| s[1].n_components | ||
| for s in transformer.steps | ||
| if isinstance(s[1], TruncatedSVD) | ||
| ) | ||
|
|
||
| @override | ||
| def _fit( | ||
| self, | ||
| X: np.ndarray, | ||
| feature_schema: FeatureSchema, | ||
| ) -> FeatureSchema: | ||
| n_samples, n_features = X.shape | ||
| if n_features < 2: | ||
| self.is_no_op = True | ||
| return feature_schema | ||
|
|
||
| static_seed, _ = infer_random_state(self.random_state) | ||
| transformer = get_svd_features_transformer( | ||
| self.global_transformer_name, | ||
| n_samples, | ||
| n_features, | ||
| random_state=static_seed, | ||
| ) | ||
| transformer.fit(X) | ||
|
|
||
| self.transformer_ = transformer | ||
| self.feature_schema_updated_ = feature_schema | ||
|
|
||
| return feature_schema | ||
|
|
||
| @override | ||
| def _transform( | ||
| self, X: np.ndarray, *, is_test: bool = False | ||
| ) -> tuple[np.ndarray, np.ndarray | None, FeatureModality | None]: | ||
| if self.is_no_op: | ||
| return X, None, None | ||
|
|
||
| assert self.feature_schema_updated_ is not None | ||
| assert self.transformer_ is not None | ||
|
|
||
| return X, self.transformer_.transform(X), FeatureModality.NUMERICAL | ||
|
|
||
|
|
||
| def get_svd_features_transformer( | ||
| global_transformer_name: Literal["svd", "svd_quarter_components"], | ||
| n_samples: int, | ||
| n_features: int, | ||
| random_state: int | None = None, | ||
| ) -> Pipeline: | ||
| """Returns a transformer to add SVD features to the data.""" | ||
| if global_transformer_name == "svd": | ||
| divisor = 2 | ||
| elif global_transformer_name == "svd_quarter_components": | ||
| divisor = 4 | ||
| else: | ||
| raise ValueError(f"Invalid global transformer name: {global_transformer_name}.") | ||
|
|
||
| n_components = max(1, min(n_samples // 10 + 1, n_features // divisor)) | ||
| return Pipeline( | ||
| steps=[ | ||
| ( | ||
| "save_standard", | ||
| make_standard_scaler_safe( | ||
| ("standard", StandardScaler(with_mean=False)), | ||
|
Contributor
There was a problem hiding this comment. @bejaeger at this step we expect many columns to already be normalized, right? and this is general is to learn a more balanced projection, but unrelated to the original features? |
||
| ), | ||
| ), | ||
| ( | ||
| "svd", | ||
| TruncatedSVD( | ||
| algorithm="arpack", | ||
| n_components=n_components, | ||
| random_state=random_state, | ||
| ), | ||
| ), | ||
| ], | ||
| ) | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What did scaler mean here?