ENH permutation importance

examples/plot_model_card.py

@@ -20,6 +20,7 @@
 from sklearn.datasets import load_breast_cancer
 from sklearn.ensemble import HistGradientBoostingClassifier
 from sklearn.experimental import enable_halving_search_cv  # noqa
+from sklearn.inspection import permutation_importance
 from sklearn.metrics import (
     ConfusionMatrixDisplay,
     accuracy_score,
@@ -148,6 +149,9 @@
 disp.figure_.savefig(Path(local_repo) / "confusion_matrix.png")
 model_card.add_plot(**{"Confusion matrix": "confusion_matrix.png"})
 
+result = permutation_importance(model, X_test, y_test, n_repeats=10)
+model_card.add_feature_importances(result)
+
 cv_results = model.cv_results_
 clf_report = classification_report(
     y_test, y_pred, output_dict=True, target_names=["malignant", "benign"]

skops/card/_model_card.py

@@ -10,6 +10,8 @@
 from reprlib import Repr
 from typing import Any, Optional, Union
 
+import matplotlib.pyplot as plt
+import pandas as pd
 from modelcards import CardData, ModelCard
 from sklearn.utils import estimator_html_repr
 from tabulate import tabulate  # type: ignore
@@ -366,6 +368,33 @@ def add_metrics(self, **kwargs: str) -> "Card":
             self._eval_results[metric] = value
         return self
 
+    def add_feature_importances(self, feature_importances) -> "Card":
+        """Visualize permutation importance.
+
+        Parameters
+        ----------
+        self : object
+            Card object.
+        feature_importances : sklearn.utils.Bunch
+            Output of sklearn.inspection.permutation_importance()
+
+        Returns
+        -------
+        self : object
+            Card object.
+        """
+
+        importances = pd.DataFrame(
+            feature_importances.importances.T,
+            columns=self.model.feature_names_in_,
+        )
+        ax = importances.plot.box(vert=False, whis=10)
+        ax.set_title("Permutation Importances")
+        ax.set_xlabel("Decrease in accuracy score")
+        plt.savefig("feature_importances.png")
+        self.add_plot(**{"Feature Importances": "feature_importances.png"})
+        return self
+
     def _generate_card(self) -> ModelCard:
         """Generate the ModelCard object