From d15b9a8309075ddbedba6fb05d84a38abb2748f9 Mon Sep 17 00:00:00 2001 From: SophieHerbst Date: Wed, 15 May 2024 11:00:49 +0200 Subject: [PATCH 1/6] save patterns and filters --- .../steps/sensor/_05_decoding_csp.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index 4f3cd0ae2..882cef3e5 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -7,10 +7,11 @@ import mne import numpy as np import pandas as pd -from mne.decoding import CSP +from mne.decoding import CSP, LinearModel from mne_bids import BIDSPath from sklearn.model_selection import StratifiedKFold, cross_val_score from sklearn.pipeline import make_pipeline +from sklearn.linear_model import LogisticRegression from ..._config_utils import ( _bids_kwargs, @@ -239,9 +240,13 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non epochs_filt, y = prepare_epochs_and_y( epochs=epochs, contrast=contrast, fmin=fmin, fmax=fmax, cfg=cfg ) + # Get the data for all time points X = epochs_filt.get_data() + + + cv_scores = cross_val_score( estimator=clf, X=X, @@ -253,6 +258,22 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non freq_decoding_table.loc[idx, "mean_crossval_score"] = cv_scores.mean() freq_decoding_table.at[idx, "scores"] = cv_scores + # PATTERNS + csp.fit_transform(X, y) + sensor_pattern_csp = csp.patterns_ + + # COEFS + clf.fit(X, y) + weights_csp = get_coef(clf, 'patterns_', inverse_transform=True) + + # save scores + # XXX right now this saves in working directory + csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + + np.save(csp_fname + '_patterns', sensor_pattern_csp) + np.save(csp_fname + '_weights', weights_csp) + + # Loop over times x frequencies # # Note: We don't support varying time ranges for different frequency @@ -306,6 +327,7 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # Crop data to the time window of interest if tmax is not None: # avoid warnings about outside the interval tmax = min(tmax, epochs_filt.times[-1]) + X = epochs_filt.crop(tmin, tmax).get_data() del epochs_filt cv_scores = cross_val_score( @@ -323,6 +345,21 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non msg += f": {cfg.decoding_metric}={score:0.3f}" logger.info(**gen_log_kwargs(msg)) + # PATTERNS + csp.fit_transform(X, y) + sensor_pattern_csp = csp.patterns_ + + # COEFS + clf.fit(X, y) + weights_csp = get_coef(clf, 'patterns_', inverse_transform=True) + + # save scores + # XXX right now this saves in working directory + csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + str(tmin) + str(tmax) + + np.save(csp_fname + '_patterns', sensor_pattern_csp) + np.save(csp_fname + '_weights', weights_csp) + # Write each DataFrame to a different Excel worksheet. a_vs_b = f"{condition1}+{condition2}".replace(op.sep, "") processing = f"{a_vs_b}+CSP+{cfg.decoding_metric}" From 43629793bc68583c121ea216263c89c97e77229e Mon Sep 17 00:00:00 2001 From: SophieHerbst Date: Wed, 15 May 2024 11:05:31 +0200 Subject: [PATCH 2/6] save patterns and filters - missing import --- mne_bids_pipeline/steps/sensor/_05_decoding_csp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index 882cef3e5..c5b177220 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -264,7 +264,7 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # COEFS clf.fit(X, y) - weights_csp = get_coef(clf, 'patterns_', inverse_transform=True) + weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) # save scores # XXX right now this saves in working directory @@ -351,7 +351,7 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # COEFS clf.fit(X, y) - weights_csp = get_coef(clf, 'patterns_', inverse_transform=True) + weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) # save scores # XXX right now this saves in working directory From 6cc2e231fe4297e06a482856da71e5f4dc04aa68 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 09:12:32 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../steps/sensor/_05_decoding_csp.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index c5b177220..55e8d56d2 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -7,11 +7,10 @@ import mne import numpy as np import pandas as pd -from mne.decoding import CSP, LinearModel +from mne.decoding import CSP from mne_bids import BIDSPath from sklearn.model_selection import StratifiedKFold, cross_val_score from sklearn.pipeline import make_pipeline -from sklearn.linear_model import LogisticRegression from ..._config_utils import ( _bids_kwargs, @@ -244,9 +243,6 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # Get the data for all time points X = epochs_filt.get_data() - - - cv_scores = cross_val_score( estimator=clf, X=X, @@ -264,15 +260,14 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # COEFS clf.fit(X, y) - weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) + weights_csp = mne.decoding.get_coef(clf, "patterns_", inverse_transform=True) # save scores # XXX right now this saves in working directory csp_fname = cond1 + cond2 + str(fmin) + str(fmax) - np.save(csp_fname + '_patterns', sensor_pattern_csp) - np.save(csp_fname + '_weights', weights_csp) - + np.save(csp_fname + "_patterns", sensor_pattern_csp) + np.save(csp_fname + "_weights", weights_csp) # Loop over times x frequencies # @@ -351,14 +346,14 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # COEFS clf.fit(X, y) - weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) + weights_csp = mne.decoding.get_coef(clf, "patterns_", inverse_transform=True) # save scores # XXX right now this saves in working directory csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + str(tmin) + str(tmax) - np.save(csp_fname + '_patterns', sensor_pattern_csp) - np.save(csp_fname + '_weights', weights_csp) + np.save(csp_fname + "_patterns", sensor_pattern_csp) + np.save(csp_fname + "_weights", weights_csp) # Write each DataFrame to a different Excel worksheet. a_vs_b = f"{condition1}+{condition2}".replace(op.sep, "") From 8732634f9074d430c8b66decb00863b3c2b7ce56 Mon Sep 17 00:00:00 2001 From: SophieHerbst Date: Wed, 15 May 2024 15:15:51 +0200 Subject: [PATCH 4/6] save patterns and filters - add linear model --- .../steps/sensor/_05_decoding_csp.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index c5b177220..7fdea9a9c 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -171,11 +171,11 @@ def one_subject_decoding( clf = make_pipeline( *preproc_steps, csp, - LogReg( + LinearModel(LogReg( solver="liblinear", # much faster than the default random_state=cfg.random_state, n_jobs=1, - ), + )), ) cv = StratifiedKFold( n_splits=cfg.decoding_n_splits, @@ -244,9 +244,6 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # Get the data for all time points X = epochs_filt.get_data() - - - cv_scores = cross_val_score( estimator=clf, X=X, @@ -258,14 +255,14 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non freq_decoding_table.loc[idx, "mean_crossval_score"] = cv_scores.mean() freq_decoding_table.at[idx, "scores"] = cv_scores - # PATTERNS - csp.fit_transform(X, y) - sensor_pattern_csp = csp.patterns_ - # COEFS clf.fit(X, y) weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) + # PATTERNS + csp.fit_transform(X, y) + sensor_pattern_csp = csp.patterns_ + # save scores # XXX right now this saves in working directory csp_fname = cond1 + cond2 + str(fmin) + str(fmax) @@ -345,14 +342,14 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non msg += f": {cfg.decoding_metric}={score:0.3f}" logger.info(**gen_log_kwargs(msg)) - # PATTERNS - csp.fit_transform(X, y) - sensor_pattern_csp = csp.patterns_ - # COEFS clf.fit(X, y) weights_csp = mne.decoding.get_coef(clf, 'patterns_', inverse_transform=True) + # PATTERNS + csp.fit_transform(X, y) + sensor_pattern_csp = csp.patterns_ + # save scores # XXX right now this saves in working directory csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + str(tmin) + str(tmax) From a0b7a06be8fc7dcf65d0e60d0d39d142e47d2bae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 13:20:15 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne_bids_pipeline/steps/sensor/_05_decoding_csp.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index 37223a487..415d67ae8 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -170,11 +170,13 @@ def one_subject_decoding( clf = make_pipeline( *preproc_steps, csp, - LinearModel(LogReg( - solver="liblinear", # much faster than the default - random_state=cfg.random_state, - n_jobs=1, - )), + LinearModel( + LogReg( + solver="liblinear", # much faster than the default + random_state=cfg.random_state, + n_jobs=1, + ) + ), ) cv = StratifiedKFold( n_splits=cfg.decoding_n_splits, @@ -261,7 +263,7 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non # PATTERNS csp.fit_transform(X, y) sensor_pattern_csp = csp.patterns_ - + # save scores # XXX right now this saves in working directory csp_fname = cond1 + cond2 + str(fmin) + str(fmax) From 92cfef9e1b9c0f92f8d9ffcb1241ffc334486469 Mon Sep 17 00:00:00 2001 From: SophieHerbst Date: Wed, 15 May 2024 15:59:20 +0200 Subject: [PATCH 6/6] write csp output to proper directory --- .../steps/sensor/_05_decoding_csp.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py index 37223a487..030720c02 100644 --- a/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py +++ b/mne_bids_pipeline/steps/sensor/_05_decoding_csp.py @@ -7,7 +7,7 @@ import mne import numpy as np import pandas as pd -from mne.decoding import CSP +from mne.decoding import CSP, LinearModel from mne_bids import BIDSPath from sklearn.model_selection import StratifiedKFold, cross_val_score from sklearn.pipeline import make_pipeline @@ -162,6 +162,10 @@ def one_subject_decoding( epochs=epochs, ) + # Create output directory if it doesn't already exist + output_dir = bids_path.fpath.parent / "CSD_output" + output_dir.mkdir(exist_ok=True) + # Classifier csp = CSP( n_components=4, # XXX revisit @@ -262,12 +266,12 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non csp.fit_transform(X, y) sensor_pattern_csp = csp.patterns_ - # save scores - # XXX right now this saves in working directory - csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + # save weights and patterns + csp_patterns_fname = f"{cond1}_{cond2}_{str(fmin)}_{str(fmax)}_Hz_patterns" + csp_weights_fname = f"{cond1}_{cond2}_{str(fmin)}_{str(fmax)}_Hz_weights" - np.save(csp_fname + "_patterns", sensor_pattern_csp) - np.save(csp_fname + "_weights", weights_csp) + np.save(op.join(output_dir, csp_patterns_fname), sensor_pattern_csp) + np.save(op.join(output_dir, csp_weights_fname), weights_csp) # Loop over times x frequencies # @@ -348,12 +352,12 @@ def _fmt_contrast(cond1, cond2, fmin, fmax, freq_range_name, tmin=None, tmax=Non csp.fit_transform(X, y) sensor_pattern_csp = csp.patterns_ - # save scores - # XXX right now this saves in working directory - csp_fname = cond1 + cond2 + str(fmin) + str(fmax) + str(tmin) + str(tmax) + # save weights and patterns + csp_patterns_fname = f"{cond1}_{cond2}_{str(fmin)}_{str(fmax)}_Hz_{str(tmin)}_{str(tmax)}_s_patterns" + csp_weights_fname = f"{cond1}_{cond2}_{str(fmin)}_{str(fmax)}_Hz_{str(tmin)}_{str(tmax)}_s_patterns" - np.save(csp_fname + "_patterns", sensor_pattern_csp) - np.save(csp_fname + "_weights", weights_csp) + np.save(op.join(output_dir, csp_patterns_fname), sensor_pattern_csp) + np.save(op.join(output_dir, csp_weights_fname), weights_csp) # Write each DataFrame to a different Excel worksheet. a_vs_b = f"{condition1}+{condition2}".replace(op.sep, "")