XAI Inverse Estimation

CONTRIBUTING.md

@@ -89,7 +89,7 @@ It is possible to limit the scope of testing to specific sections of the codebas
 Faithfulness metrics using python3.9 (make sure the python versions match in your environment):
 
 ```bash
-python3 -m tox run -e py39 -- -m evaluate_func -s
+python3 -m tox run -e py39 -- -m faithfulness -s
 ```
 
 For a complete overview of the possible testing scopes, please refer to `pytest.ini`.

quantus/functions/explanation_func.py

@@ -671,7 +671,7 @@ def f_reduce_axes(a):
             inputs = inputs.cpu()
 
         inputs_numpy = inputs.detach().numpy()
-                
+
         for i in range(len(explanation)):
             explanation[i] = torch.Tensor(
                 np.clip(scipy.ndimage.sobel(inputs_numpy[i]), 0, 1)

quantus/helpers/constants.py

@@ -27,7 +27,7 @@
         "Faithfulness Correlation": FaithfulnessCorrelation,
         "Faithfulness Estimate": FaithfulnessEstimate,
         "Pixel-Flipping": PixelFlipping,
-        "Region Segmentation": RegionPerturbation,
+        "Region Perturbation": RegionPerturbation,
         "Monotonicity-Arya": Monotonicity,
         "Monotonicity-Nguyen": MonotonicityCorrelation,
         "Selectivity": Selectivity,
@@ -74,6 +74,15 @@
     },
 }
 
+# Quantus metrics that include a step-wise 'masking'/ perturbation that is
+# based on attribution order/ ranking (and not magnitude).
+AVAILABLE_INVERSE_ESTIMATION_METRICS = {
+    "Pixel-Flipping": PixelFlipping,
+    "Region Perturbation": RegionPerturbation,  # order = 'morf'
+    "ROAD": ROAD,  # return_only_values = True
+    "Selectivity": Selectivity,
+}
+#
 
 AVAILABLE_PERTURBATION_FUNCTIONS = {
     "baseline_replacement_by_indices": baseline_replacement_by_indices,

quantus/helpers/model/pytorch_model.py

@@ -362,6 +362,38 @@ def sample(
                         )
         return model_copy
 
+    def perturb_layer_weights(self, layer_idx: int, noise: float):
+        """
+        Perturb the weights of a specific layer in a PyTorch model.
+
+        Parameters
+        ----------
+        model : torch.nn.Module
+            The PyTorch model.
+        layer_idx : int
+            The index of the layer to perturb.
+        noise : float
+            The standard deviation of the Gaussian noise to add to the weights.
+
+        Returns
+        -------
+        None
+        """
+        original_parameters = self.state_dict()
+        model_copy = deepcopy(self.model)
+        model_copy.load_state_dict(original_parameters)
+
+        # Get the specific layer.
+        layer = list(model_copy.modules())[layer_idx]
+
+        # Generate Gaussian noise.
+        noise_tensor = torch.randn_like(layer.weight) * noise
+
+        # Add the noise to the layer's weights.
+        layer.weight.data.add_(noise_tensor)
+
+        return model_copy
+
     def add_mean_shift_to_first_layer(
         self,
         input_shift: Union[int, float],

quantus/helpers/perturbation_utils.py

@@ -28,7 +28,23 @@ def __call__(
 def make_perturb_func(
     perturb_func: PerturbFunc, perturb_func_kwargs: Mapping[str, ...] | None, **kwargs
 ) -> PerturbFunc | functools.partial:
-    """A utility function to save few lines of code during perturbation metric initialization."""
+    """
+    A utility function to save few lines of code during perturbation metric initialization.
+
+    Parameters
+    ----------
+    perturb_func: callable
+        Perturbation function.
+    perturb_func_kwargs: dict
+        Perturbation function kwargs.
+    kwargs: dict
+        Perturbation metric kwargs.
+
+    Returns
+    -------
+    perturb_func: callable
+        Perturbation function.
+    """
     if perturb_func_kwargs is not None:
         func_kwargs = kwargs.copy()
         func_kwargs.update(perturb_func_kwargs)
@@ -41,7 +57,19 @@ def make_perturb_func(
 def make_changed_prediction_indices_func(
     return_nan_when_prediction_changes: bool,
 ) -> Callable[[ModelInterface, np.ndarray, np.ndarray], List[int]]:
-    """A utility function to improve static analysis."""
+    """
+    A utility function to improve static analysis.
+
+    Parameters
+    ----------
+    return_nan_when_prediction_changes: boolean
+        Indicates if metric should return NaN when model prediction changes due to perturbation.
+
+    Returns
+    -------
+    changed_prediction_indices: callable
+        Function that returns indices in batch, for which predicted label has changed after applying perturbation.
+    """
     return functools.partial(
         changed_prediction_indices,
         return_nan_when_prediction_changes=return_nan_when_prediction_changes,
@@ -62,15 +90,15 @@ def changed_prediction_indices(
     ----------
     return_nan_when_prediction_changes:
         Instance attribute of perturbation metrics.
-    model:
+    model: ModelInterface
+        Model to be used for prediction.
     x_batch:
         Batch of original inputs provided by user.
     x_perturbed:
         Batch of inputs after applying perturbation.
 
     Returns
     -------
-
     changed_idx:
         List of indices in batch, for which predicted label has changed afer.
 

quantus/metrics/__init__.py

@@ -11,3 +11,4 @@
 from quantus.metrics.localisation import *
 from quantus.metrics.randomisation import *
 from quantus.metrics.robustness import *
+from quantus.metrics.inverse_estimation import InverseEstimation

quantus/metrics/axiomatic/completeness.py

@@ -219,32 +219,32 @@ def __call__(
         Examples:
         --------
             # Minimal imports.
-            >> import quantus
-            >> from quantus import LeNet
-            >> import torch
+            >>> import quantus
+            >>> from quantus import LeNet
+            >>> import torch
 
             # Enable GPU.
-            >> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            >>> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
             # Load a pre-trained LeNet classification model (architecture at quantus/helpers/models).
-            >> model = LeNet()
-            >> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
+            >>> model = LeNet()
+            >>> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
 
             # Load MNIST datasets and make loaders.
-            >> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
-            >> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
+            >>> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
+            >>> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
 
             # Load a batch of inputs and outputs to use for XAI evaluation.
-            >> x_batch, y_batch = iter(test_loader).next()
-            >> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
+            >>> x_batch, y_batch = iter(test_loader).next()
+            >>> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
 
             # Generate Saliency attributions of the test set batch of the test set.
-            >> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
-            >> a_batch_saliency = a_batch_saliency.cpu().numpy()
+            >>> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
+            >>> a_batch_saliency = a_batch_saliency.cpu().numpy()
 
             # Initialise the metric and evaluate explanations by calling the metric instance.
-            >> metric = Metric(abs=True, normalise=False)
-            >> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
+            >>> metric = Metric(abs=True, normalise=False)
+            >>> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
         """
         return super().__call__(
             model=model,

quantus/metrics/axiomatic/input_invariance.py

@@ -199,32 +199,32 @@ def __call__(
         Examples:
         --------
             # Minimal imports.
-            >> import quantus
-            >> from quantus import LeNet
-            >> import torch
+            >>> import quantus
+            >>> from quantus import LeNet
+            >>> import torch
 
             # Enable GPU.
-            >> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            >>> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
             # Load a pre-trained LeNet classification model (architecture at quantus/helpers/models).
-            >> model = LeNet()
-            >> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
+            >>> model = LeNet()
+            >>> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
 
             # Load MNIST datasets and make loaders.
-            >> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
-            >> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
+            >>> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
+            >>> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
 
             # Load a batch of inputs and outputs to use for XAI evaluation.
-            >> x_batch, y_batch = iter(test_loader).next()
-            >> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
+            >>> x_batch, y_batch = iter(test_loader).next()
+            >>> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
 
             # Generate Saliency attributions of the test set batch of the test set.
-            >> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
-            >> a_batch_saliency = a_batch_saliency.cpu().numpy()
+            >>> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
+            >>> a_batch_saliency = a_batch_saliency.cpu().numpy()
 
             # Initialise the metric and evaluate explanations by calling the metric instance.
-            >> metric = Metric(abs=True, normalise=False)
-            >> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
+            >>> metric = Metric(abs=True, normalise=False)
+            >>> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
         """
         return super().__call__(
             model=model,

quantus/metrics/axiomatic/non_sensitivity.py

@@ -219,32 +219,32 @@ def __call__(
         Examples:
         --------
             # Minimal imports.
-            >> import quantus
-            >> from quantus import LeNet
-            >> import torch
+            >>> import quantus
+            >>> from quantus import LeNet
+            >>> import torch
 
             # Enable GPU.
-            >> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            >>> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
             # Load a pre-trained LeNet classification model (architecture at quantus/helpers/models).
-            >> model = LeNet()
-            >> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
+            >>> model = LeNet()
+            >>> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
 
             # Load MNIST datasets and make loaders.
-            >> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
-            >> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
+            >>> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
+            >>> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
 
             # Load a batch of inputs and outputs to use for XAI evaluation.
-            >> x_batch, y_batch = iter(test_loader).next()
-            >> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
+            >>> x_batch, y_batch = iter(test_loader).next()
+            >>> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
 
             # Generate Saliency attributions of the test set batch of the test set.
-            >> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
-            >> a_batch_saliency = a_batch_saliency.cpu().numpy()
+            >>> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
+            >>> a_batch_saliency = a_batch_saliency.cpu().numpy()
 
             # Initialise the metric and evaluate explanations by calling the metric instance.
-            >> metric = Metric(abs=True, normalise=False)
-            >> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
+            >>> metric = Metric(abs=True, normalise=False)
+            >>> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
         """
         return super().__call__(
             model=model,

quantus/metrics/base.py

@@ -140,7 +140,7 @@ def __init__(
 
         if normalise_func_kwargs is not None:
             normalise_func = functools.partial(normalise_func, **normalise_func_kwargs)
-
+        
         # Run deprecation warnings.
         warn.deprecation_warnings(kwargs)
         warn.check_kwargs(kwargs)
@@ -233,32 +233,32 @@ def __call__(
         Examples:
         --------
             # Minimal imports.
-            >> import quantus
-            >> from quantus import LeNet
-            >> import torch
+            >>> import quantus
+            >>> from quantus import LeNet
+            >>> import torch
 
             # Enable GPU.
-            >> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            >>> device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
             # Load a pre-trained LeNet classification model (architecture at quantus/helpers/models).
-            >> model = LeNet()
-            >> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
+            >>> model = LeNet()
+            >>> model.load_state_dict(torch.load("tutorials/assets/pytests/mnist_model"))
 
             # Load MNIST datasets and make loaders.
-            >> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
-            >> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
+            >>> test_set = torchvision.datasets.MNIST(root='./sample_data', download=True)
+            >>> test_loader = torch.utils.data.DataLoader(test_set, batch_size=24)
 
             # Load a batch of inputs and outputs to use for XAI evaluation.
-            >> x_batch, y_batch = iter(test_loader).next()
-            >> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
+            >>> x_batch, y_batch = iter(test_loader).next()
+            >>> x_batch, y_batch = x_batch.cpu().numpy(), y_batch.cpu().numpy()
 
             # Generate Saliency attributions of the test set batch of the test set.
-            >> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
-            >> a_batch_saliency = a_batch_saliency.cpu().numpy()
+            >>> a_batch_saliency = Saliency(model).attribute(inputs=x_batch, target=y_batch, abs=True).sum(axis=1)
+            >>> a_batch_saliency = a_batch_saliency.cpu().numpy()
 
             # Initialise the metric and evaluate explanations by calling the metric instance.
-            >> metric = Metric(abs=True, normalise=False)
-            >> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
+            >>> metric = Metric(abs=True, normalise=False)
+            >>> scores = metric(model=model, x_batch=x_batch, y_batch=y_batch, a_batch=a_batch_saliency)
         """
         # Run deprecation warnings.
         warn.deprecation_warnings(kwargs)