From e5c363d72a17f58e3e3867f2e0bee7dbd2e9fb65 Mon Sep 17 00:00:00 2001
From: Henrik <89074178+henrikfo@users.noreply.github.com>
Date: Wed, 18 Dec 2024 10:28:07 +0100
Subject: [PATCH] Fixed broken pdf  (#194)

* fixed some pdf stuff

* fixes tests

* ..
---
 examples/report_handler/report_handler.ipynb  | 397 ++----------------
 leakpro/metrics/attack_result.py              |  23 +-
 leakpro/reporting/report_handler.py           |  14 +
 .../test_attack_result/test_attack_result.py  |  15 +-
 4 files changed, 69 insertions(+), 380 deletions(-)

diff --git a/examples/report_handler/report_handler.ipynb b/examples/report_handler/report_handler.ipynb
index 94cae90..100067c 100644
--- a/examples/report_handler/report_handler.ipynb
+++ b/examples/report_handler/report_handler.ipynb
@@ -239,319 +239,7 @@
    "execution_count": null,
    "id": "f28eb14f",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2024-12-11 13:30:05,252 INFO     Target model blueprint created from ResNet18 in ./mia_utils/utils/cifar_model_preparation.py.\n",
-      "2024-12-11 13:30:05,254 INFO     Loaded target model metadata from ./target/model_metadata.pkl\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'audit': {'random_seed': 1234, 'attack_list': {'rmia': {'training_data_fraction': 0.5, 'attack_data_fraction': 0.5, 'num_shadow_models': 3, 'online': True, 'temperature': 2, 'gamma': 2.0, 'offline_a': 0.33, 'offline_b': 0.66}, 'population': {'attack_data_fraction': 1.0}, 'lira': {'training_data_fraction': 0.5, 'num_shadow_models': 3, 'online': True}, 'loss_traj': {'training_distill_data_fraction': 0.7, 'number_of_traj': 10, 'label_only': False, 'mia_classifier_epochs': 100}, 'HSJ': {'attack_data_fraction': 0.01, 'target_metadata_path': './target/model_metadata.pkl', 'num_iterations': 2, 'initial_num_evals': 100, 'max_num_evals': 10000, 'stepsize_search': 'geometric_progression', 'gamma': 1.0, 'constraint': 2, 'batch_size': 50, 'verbose': True, 'epsilon_threshold': '1e-6'}}, 'output_dir': './leakpro_output', 'attack_type': 'mia', 'modality': 'image'}, 'target': {'module_path': './mia_utils/utils/cifar_model_preparation.py', 'model_class': 'ResNet18', 'target_folder': './target', 'data_path': './data/cifar10.pkl'}, 'shadow_model': None, 'distillation_model': None}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
-      "  warnings.warn(\n",
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
-      "  warnings.warn(msg)\n",
-      "2024-12-11 13:30:05,518 INFO     Loaded target model from ./target\n",
-      "2024-12-11 13:30:06,417 INFO     Loaded population dataset from ./data/cifar10.pkl\n",
-      "2024-12-11 13:30:06,419 INFO     Loaded population dataset from ./data/cifar10.pkl\n",
-      "2024-12-11 13:30:06,420 INFO     Creating shadow model handler singleton\n",
-      "2024-12-11 13:30:06,423 INFO     Creating distillation model handler singleton\n",
-      "2024-12-11 13:30:06,425 INFO     Configuring RMIA attack\n",
-      "2024-12-11 13:30:06,426 INFO     Added attack: rmia\n",
-      "2024-12-11 13:30:06,427 INFO     Configuring the Population attack\n",
-      "2024-12-11 13:30:06,428 INFO     Added attack: population\n",
-      "2024-12-11 13:30:06,429 INFO     Added attack: lira\n",
-      "2024-12-11 13:30:06,430 INFO     Configuring Loss trajecatory attack\n",
-      "2024-12-11 13:30:06,434 INFO     Added attack: loss_traj\n",
-      "2024-12-11 13:30:06,436 INFO     Configuring label only attack\n",
-      "2024-12-11 13:30:06,438 INFO     Added attack: HSJ\n",
-      "2024-12-11 13:30:06,439 INFO     Preparing attack: rmia\n",
-      "2024-12-11 13:30:06,440 INFO     Preparing shadow models for RMIA attack\n",
-      "2024-12-11 13:30:06,441 INFO     Preparing attack data for training the RMIA attack\n",
-      "2024-12-11 13:30:06,447 INFO     Check for 3 shadow models (dataset: 60000 points)\n",
-      "2024-12-11 13:30:07,605 INFO     Number of existing models exceeds or equals the number of models to create\n",
-      "2024-12-11 13:30:07,607 INFO     Loading shadow model 2\n",
-      "2024-12-11 13:30:07,832 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_2.pkl\n",
-      "2024-12-11 13:30:07,833 INFO     Loading shadow model 1\n",
-      "2024-12-11 13:30:08,059 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_1.pkl\n",
-      "2024-12-11 13:30:08,060 INFO     Loading shadow model 0\n",
-      "2024-12-11 13:30:08,274 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_0.pkl\n",
-      "2024-12-11 13:30:08,279 INFO     Running attack: rmia\n",
-      "2024-12-11 13:30:08,280 INFO     Running RMIA online attack\n",
-      "2024-12-11 13:30:08,281 INFO     Loading metadata 2\n",
-      "2024-12-11 13:30:08,285 INFO     Loading metadata 1\n",
-      "2024-12-11 13:30:08,287 INFO     Loading metadata 0\n",
-      "2024-12-11 13:30:08,881 INFO     Number of points in the audit dataset that are used for online attack: 26907\n",
-      "2024-12-11 13:31:51,245 INFO     Subsampling attack data from 24000 points      \n",
-      "2024-12-11 13:31:51,249 INFO     Number of attack data points after subsampling: 12000\n",
-      "2024-12-11 13:32:41,414 INFO     Finished attack: rmia                          \n",
-      "2024-12-11 13:32:41,416 INFO     Preparing attack: population\n",
-      "2024-12-11 13:32:41,417 INFO     Preparing attack data for training the Population attack\n",
-      "2024-12-11 13:32:41,423 INFO     Subsampling attack data from 24000 points\n",
-      "2024-12-11 13:32:41,425 INFO     Number of attack data points after subsampling: 24000\n",
-      "2024-12-11 13:32:41,426 INFO     Computing signals for the Population attack\n",
-      "Getting loss for model 1/ 1: 100%|██████████| 750/750 [00:13<00:00, 57.54it/s]\n",
-      "2024-12-11 13:32:54,560 INFO     Running attack: population\n",
-      "2024-12-11 13:32:54,565 INFO     Running the Population attack on the target model\n",
-      "Getting loss for model 1/ 1: 100%|██████████| 1125/1125 [00:17<00:00, 64.57it/s]\n",
-      "2024-12-11 13:33:12,093 INFO     Attack completed\n",
-      "2024-12-11 13:33:12,105 INFO     Finished attack: population\n",
-      "2024-12-11 13:33:12,106 INFO     Preparing attack: lira\n",
-      "2024-12-11 13:33:12,131 INFO     Number of existing models exceeds or equals the number of models to create\n",
-      "2024-12-11 13:33:12,132 INFO     Loading shadow model 2\n",
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
-      "  warnings.warn(\n",
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
-      "  warnings.warn(msg)\n",
-      "2024-12-11 13:33:12,340 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_2.pkl\n",
-      "2024-12-11 13:33:12,341 INFO     Loading shadow model 1\n",
-      "2024-12-11 13:33:12,543 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_1.pkl\n",
-      "2024-12-11 13:33:12,544 INFO     Loading shadow model 0\n",
-      "2024-12-11 13:33:12,745 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_0.pkl\n",
-      "2024-12-11 13:33:12,748 INFO     Create masks for all IN and OUT samples\n",
-      "2024-12-11 13:33:12,749 INFO     Loading metadata 2\n",
-      "2024-12-11 13:33:12,750 INFO     Loading metadata 1\n",
-      "2024-12-11 13:33:12,752 INFO     Loading metadata 0\n",
-      "2024-12-11 13:33:12,761 INFO     Calculating the logits for all 3 shadow models\n",
-      "2024-12-11 13:34:25,365 INFO     Calculating the logits for the target model             \n",
-      "2024-12-11 13:34:51,520 INFO     Running attack: lira                                    \n",
-      "Processing audit samples: 100%|██████████| 26907/26907 [00:06<00:00, 4483.96it/s]\n",
-      "2024-12-11 13:34:57,628 INFO     Finished attack: lira\n",
-      "2024-12-11 13:34:57,630 INFO     Preparing attack: loss_traj\n",
-      "2024-12-11 13:34:57,631 INFO     Preparing the data for loss trajectory attack\n",
-      "2024-12-11 13:34:57,640 INFO     Training shadow models on 3600 points\n",
-      "2024-12-11 13:34:57,647 INFO     Number of existing models exceeds or equals the number of models to create\n",
-      "2024-12-11 13:34:57,648 INFO     Loading shadow model 6\n",
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
-      "  warnings.warn(\n",
-      "/opt/conda/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
-      "  warnings.warn(msg)\n",
-      "2024-12-11 13:34:57,858 INFO     Loaded model from ./leakpro_output/attack_objects/shadow_model/shadow_model_6.pkl\n",
-      "2024-12-11 13:34:58,208 INFO     Training distillation of the shadow model on 21147 points\n",
-      "2024-12-11 13:34:58,276 INFO     Created distillation dataset with size 21147\n",
-      "Epoch 1/10: 100%|██████████| 166/166 [00:07<00:00, 21.51it/s]\n",
-      "2024-12-11 13:35:05,996 INFO     Epoch 1/10 | Loss: 63.67065370082855\n",
-      "2024-12-11 13:35:06,119 INFO     Saved distillation model for epoch 0 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:06,121 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:06,126 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 2/10: 100%|██████████| 166/166 [00:07<00:00, 21.17it/s]\n",
-      "2024-12-11 13:35:13,972 INFO     Epoch 2/10 | Loss: 9.804248925298452\n",
-      "2024-12-11 13:35:14,085 INFO     Saved distillation model for epoch 1 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:14,087 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:14,091 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 3/10: 100%|██████████| 166/166 [00:07<00:00, 20.81it/s]\n",
-      "2024-12-11 13:35:22,074 INFO     Epoch 3/10 | Loss: 5.905309362336993\n",
-      "2024-12-11 13:35:22,189 INFO     Saved distillation model for epoch 2 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:22,191 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:22,194 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 4/10: 100%|██████████| 166/166 [00:08<00:00, 19.90it/s]\n",
-      "2024-12-11 13:35:30,539 INFO     Epoch 4/10 | Loss: 4.59359712805599\n",
-      "2024-12-11 13:35:30,672 INFO     Saved distillation model for epoch 3 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:30,674 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:30,678 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 5/10: 100%|██████████| 166/166 [00:08<00:00, 19.77it/s]\n",
-      "2024-12-11 13:35:39,080 INFO     Epoch 5/10 | Loss: 4.146416590549052\n",
-      "2024-12-11 13:35:39,197 INFO     Saved distillation model for epoch 4 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:39,198 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:39,202 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 6/10: 100%|██████████| 166/166 [00:08<00:00, 19.94it/s]\n",
-      "2024-12-11 13:35:47,532 INFO     Epoch 6/10 | Loss: 3.597899131476879\n",
-      "2024-12-11 13:35:47,654 INFO     Saved distillation model for epoch 5 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:47,656 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:47,660 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 7/10: 100%|██████████| 166/166 [00:07<00:00, 20.79it/s]\n",
-      "2024-12-11 13:35:55,650 INFO     Epoch 7/10 | Loss: 3.298955911770463\n",
-      "2024-12-11 13:35:55,765 INFO     Saved distillation model for epoch 6 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:35:55,767 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:35:55,771 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 8/10: 100%|██████████| 166/166 [00:07<00:00, 21.17it/s]\n",
-      "2024-12-11 13:36:03,616 INFO     Epoch 8/10 | Loss: 3.090168266557157\n",
-      "2024-12-11 13:36:03,730 INFO     Saved distillation model for epoch 7 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:03,731 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:03,736 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 9/10: 100%|██████████| 166/166 [00:07<00:00, 21.07it/s]\n",
-      "2024-12-11 13:36:11,618 INFO     Epoch 9/10 | Loss: 2.8429356180131435\n",
-      "2024-12-11 13:36:11,731 INFO     Saved distillation model for epoch 8 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:11,732 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:11,736 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 10/10: 100%|██████████| 166/166 [00:07<00:00, 21.02it/s]\n",
-      "2024-12-11 13:36:19,639 INFO     Epoch 10/10 | Loss: 2.593701013363898\n",
-      "2024-12-11 13:36:19,759 INFO     Saved distillation model for epoch 9 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:19,760 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:19,764 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:19,914 INFO     Created distillation dataset with size 21147\n",
-      "Epoch 1/10: 100%|██████████| 166/166 [00:07<00:00, 21.29it/s]\n",
-      "2024-12-11 13:36:27,717 INFO     Epoch 1/10 | Loss: 114.02811643481255\n",
-      "2024-12-11 13:36:27,829 INFO     Saved distillation model for epoch 0 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:27,831 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:27,837 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 2/10: 100%|██████████| 166/166 [00:07<00:00, 20.76it/s]\n",
-      "2024-12-11 13:36:35,838 INFO     Epoch 2/10 | Loss: 44.576862797141075\n",
-      "2024-12-11 13:36:35,950 INFO     Saved distillation model for epoch 1 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:35,951 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:35,956 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 3/10: 100%|██████████| 166/166 [00:07<00:00, 20.76it/s]\n",
-      "2024-12-11 13:36:43,958 INFO     Epoch 3/10 | Loss: 28.83185875415802\n",
-      "2024-12-11 13:36:44,077 INFO     Saved distillation model for epoch 2 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:44,079 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:44,083 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 4/10: 100%|██████████| 166/166 [00:08<00:00, 20.59it/s]\n",
-      "2024-12-11 13:36:52,150 INFO     Epoch 4/10 | Loss: 21.28936092555523\n",
-      "2024-12-11 13:36:52,270 INFO     Saved distillation model for epoch 3 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:36:52,272 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:36:52,277 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 5/10: 100%|██████████| 166/166 [00:08<00:00, 20.58it/s]\n",
-      "2024-12-11 13:37:00,346 INFO     Epoch 5/10 | Loss: 16.496566824615\n",
-      "2024-12-11 13:37:00,457 INFO     Saved distillation model for epoch 4 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:00,459 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:00,463 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 6/10: 100%|██████████| 166/166 [00:08<00:00, 20.54it/s]\n",
-      "2024-12-11 13:37:08,548 INFO     Epoch 6/10 | Loss: 13.477496419101954\n",
-      "2024-12-11 13:37:08,660 INFO     Saved distillation model for epoch 5 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:08,662 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:08,666 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 7/10: 100%|██████████| 166/166 [00:08<00:00, 20.69it/s]\n",
-      "2024-12-11 13:37:16,693 INFO     Epoch 7/10 | Loss: 11.912189535796642\n",
-      "2024-12-11 13:37:16,812 INFO     Saved distillation model for epoch 6 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:16,814 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:16,818 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 8/10: 100%|██████████| 166/166 [00:08<00:00, 20.62it/s]\n",
-      "2024-12-11 13:37:24,872 INFO     Epoch 8/10 | Loss: 11.11345386132598\n",
-      "2024-12-11 13:37:24,991 INFO     Saved distillation model for epoch 7 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:24,992 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:24,997 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 9/10: 100%|██████████| 166/166 [00:08<00:00, 20.44it/s]\n",
-      "2024-12-11 13:37:33,121 INFO     Epoch 9/10 | Loss: 10.197872441262007\n",
-      "2024-12-11 13:37:33,241 INFO     Saved distillation model for epoch 8 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:33,242 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:33,247 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "Epoch 10/10: 100%|██████████| 166/166 [00:08<00:00, 20.26it/s]\n",
-      "2024-12-11 13:37:41,445 INFO     Epoch 10/10 | Loss: 9.438245516270399\n",
-      "2024-12-11 13:37:41,565 INFO     Saved distillation model for epoch 9 to ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:41,566 INFO     Storing metadata for distillation model\n",
-      "2024-12-11 13:37:41,570 INFO     Metadata for distillation model stored in ./leakpro_output/attack_objects/distillation_model\n",
-      "2024-12-11 13:37:41,595 INFO     Loading MIA trajectory_train_data.pkl: 7200 points\n",
-      "2024-12-11 13:37:41,602 INFO     Loading MIA trajectory_test_data.pkl: 36000 points\n",
-      "2024-12-11 13:37:41,610 INFO     Running attack: loss_traj\n",
-      "2024-12-11 13:37:41,614 INFO     Loading Loss Trajectory classifier\n",
-      "2024-12-11 13:37:41,615 INFO     Running the MIA attack\n",
-      "100%|██████████| 563/563 [00:00<00:00, 823.90it/s]\n",
-      "2024-12-11 13:37:42,932 INFO     Finished attack: loss_traj\n",
-      "2024-12-11 13:37:42,933 INFO     Preparing attack: HSJ\n",
-      "2024-12-11 13:37:42,934 INFO     Preparing the data for Hop Skip Jump attack\n",
-      "2024-12-11 13:37:42,938 INFO     Running attack: HSJ\n",
-      "2024-12-11 13:37:42,939 INFO     Running Hop Skip Jump distance attack\n",
-      "Epoch: 100%|██████████| 8/8 [00:00<00:00, 131.66it/s]\n",
-      "2024-12-11 13:37:44,287 INFO     All data points in the batch have been successfully perturbed by random noise after 20 evaluations.\n",
-      "Batch:   0%|          | 0/8 [00:00<?, ?it/s]2024-12-11 13:37:50,054 INFO     Batch 0 Initial distance: 17.41651725769043 per sample\n",
-      "2024-12-11 13:38:03,481 INFO     iteration: 1, avg. distance 24.543710708618164 per sample\n",
-      "2024-12-11 13:38:21,485 INFO     iteration: 2, avg. distance 18.286640167236328 per sample\n",
-      "2024-12-11 13:38:21,497 INFO     iteration: 2, avg. distance tensor([39.9760, 13.8336, 11.5957, 23.7214,  7.0036,  8.8674, 14.1103, 32.9877,\n",
-      "        53.4290, 15.1449, 15.3655,  5.4556, 39.8997, 28.0956, 14.4931,  6.8428,\n",
-      "        17.2583, 32.1637, 18.5948, 18.7210, 25.7094, 11.3050, 26.5756,  6.9951,\n",
-      "        10.5423, 25.9252,  0.0623,  1.3028,  9.8505,  0.5891, 11.0744, 32.8501,\n",
-      "        14.2105, 13.7788, 14.4174, 40.1237, 45.3766, 29.7482,  9.6849, 17.6070,\n",
-      "        13.5251,  3.1167, 31.2205, 14.1691,  0.6734, 15.5564, 26.3781,  4.0809,\n",
-      "        16.4932, 23.8301], device='cuda:0')\n",
-      "2024-12-11 13:38:21,498 INFO     Batch 0 \n",
-      "Batch:  12%|█▎        | 1/8 [00:31<03:40, 31.48s/it]2024-12-11 13:38:21,502 INFO     Batch 1 Initial distance: 19.93766975402832 per sample\n",
-      "2024-12-11 13:38:34,873 INFO     iteration: 1, avg. distance 25.315149307250977 per sample\n",
-      "2024-12-11 13:38:52,650 INFO     iteration: 2, avg. distance 19.601409912109375 per sample\n",
-      "2024-12-11 13:38:52,653 INFO     iteration: 2, avg. distance tensor([17.4292,  6.6666,  1.5603, 19.8040,  0.7222, 29.7677,  0.6398, 29.5402,\n",
-      "        22.7341,  3.7507, 11.4965, 13.8460, 20.6868,  5.7909, 28.7771, 24.5910,\n",
-      "        20.3703, 24.2568, 41.6447, 25.2147, 35.0476, 24.9270, 29.9442, 22.7737,\n",
-      "        15.4492, 32.6298, 30.3562, 12.6462, 12.9032, 13.5410, 28.1122,  7.7441,\n",
-      "        27.9051,  4.6589, 23.4709, 37.9872, 13.3983,  0.8158, 15.9960,  8.9793,\n",
-      "         7.0445, 37.6089, 31.4072,  5.3228, 41.4638, 28.6488, 27.8128, 18.3170,\n",
-      "        19.0672, 14.8020], device='cuda:0')\n",
-      "2024-12-11 13:38:52,654 INFO     Batch 1 \n",
-      "Batch:  25%|██▌       | 2/8 [01:02<03:07, 31.29s/it]2024-12-11 13:38:52,658 INFO     Batch 2 Initial distance: 17.89975357055664 per sample\n",
-      "2024-12-11 13:39:05,906 INFO     iteration: 1, avg. distance 24.353178024291992 per sample\n",
-      "2024-12-11 13:39:23,770 INFO     iteration: 2, avg. distance 19.867765426635742 per sample\n",
-      "2024-12-11 13:39:23,774 INFO     iteration: 2, avg. distance tensor([4.0375e-02, 2.1032e+00, 3.0603e+01, 2.5822e+01, 2.4760e+01, 4.6295e+00,\n",
-      "        5.5044e-01, 2.1732e+01, 1.1121e+01, 7.2296e+00, 4.0459e+01, 3.0140e+01,\n",
-      "        4.4992e+01, 3.9899e+00, 6.8557e+00, 7.1007e+01, 3.8754e+01, 1.5153e+01,\n",
-      "        4.3451e+01, 2.5383e+01, 2.9470e+00, 5.2642e+01, 3.3935e+01, 2.5673e+01,\n",
-      "        1.1335e+01, 3.7421e+01, 1.8205e+01, 2.5378e+01, 5.4038e+00, 3.1124e+01,\n",
-      "        2.5530e+01, 1.7824e+01, 3.0967e+01, 1.0558e+01, 2.4639e+01, 1.1856e+01,\n",
-      "        4.7600e+00, 4.4461e+00, 2.6642e+01, 5.0276e+01, 1.3863e+01, 1.6391e+01,\n",
-      "        1.0539e+00, 1.3491e+00, 3.3080e+00, 2.1427e+01, 1.3908e+00, 2.3896e+01,\n",
-      "        2.0081e+00, 8.3623e+00], device='cuda:0')\n",
-      "2024-12-11 13:39:23,776 INFO     Batch 2 \n",
-      "Batch:  38%|███▊      | 3/8 [01:33<02:36, 31.21s/it]2024-12-11 13:39:23,780 INFO     Batch 3 Initial distance: 17.481121063232422 per sample\n",
-      "2024-12-11 13:39:37,194 INFO     iteration: 1, avg. distance 23.322500228881836 per sample\n",
-      "2024-12-11 13:39:54,981 INFO     iteration: 2, avg. distance 16.06374740600586 per sample\n",
-      "2024-12-11 13:39:54,986 INFO     iteration: 2, avg. distance tensor([ 5.6400, 23.0334, 12.7915, 17.6103, 28.7076,  9.6220, 15.1711,  8.0970,\n",
-      "        16.6746, 11.8406,  2.7199,  1.7019, 19.8973,  4.6822, 24.7064, 17.1194,\n",
-      "        15.0478,  7.4836, 24.4165, 32.2530,  3.6263,  9.6531, 30.7607,  1.3062,\n",
-      "         9.8290, 18.9514, 10.2762,  1.0457, 17.4370, 16.3652,  3.6932,  0.7170,\n",
-      "        30.4616, 14.9974, 19.6263,  5.0733, 14.9135, 12.8476, 15.0159, 24.5560,\n",
-      "         3.6163, 76.3436, 10.6100,  5.7315, 29.9055, 26.0178, 27.1875,  3.6305,\n",
-      "        14.7159, 45.0600], device='cuda:0')\n",
-      "2024-12-11 13:39:54,988 INFO     Batch 3 \n",
-      "Batch:  50%|█████     | 4/8 [02:04<02:04, 31.21s/it]2024-12-11 13:39:54,992 INFO     Batch 4 Initial distance: 21.497217178344727 per sample\n",
-      "2024-12-11 13:40:08,297 INFO     iteration: 1, avg. distance 29.362457275390625 per sample\n",
-      "2024-12-11 13:40:26,163 INFO     iteration: 2, avg. distance 21.2553653717041 per sample\n",
-      "2024-12-11 13:40:26,167 INFO     iteration: 2, avg. distance tensor([46.4360,  8.9194,  4.0803,  7.4007,  0.8735, 16.5769, 33.6603, 45.6845,\n",
-      "        43.7740,  4.2307, 16.5470, 23.5691, 39.7116,  9.7974, 19.9740,  2.7862,\n",
-      "         8.3742, 20.9659,  0.3752, 31.7477, 42.7279, 23.2351,  0.9155, 10.9438,\n",
-      "        29.8315, 28.9924, 30.2538, 27.8850, 48.8655,  9.2427,  5.9468, 13.9965,\n",
-      "        11.1957, 11.8729, 28.6268,  9.2214, 15.9329,  2.4883,  0.9729,  5.9621,\n",
-      "         7.7504, 25.2635, 47.0382, 31.9497, 14.2718, 15.1471, 59.3811, 50.6140,\n",
-      "        34.3203, 32.4382], device='cuda:0')\n",
-      "2024-12-11 13:40:26,168 INFO     Batch 4 \n",
-      "Batch:  62%|██████▎   | 5/8 [02:36<01:33, 31.20s/it]2024-12-11 13:40:26,172 INFO     Batch 5 Initial distance: 17.28066635131836 per sample\n",
-      "2024-12-11 13:40:39,565 INFO     Batch 5,data point 6, iter 4: epsilon is less than threshold\n",
-      "2024-12-11 13:40:39,568 INFO     iteration: 1, avg. distance 24.338886260986328 per sample\n",
-      "2024-12-11 13:40:57,427 INFO     iteration: 2, avg. distance 17.99785041809082 per sample\n",
-      "2024-12-11 13:40:57,430 INFO     iteration: 2, avg. distance tensor([ 0.9155, 15.6311, 17.4986, 22.5857, 26.1791,  0.3057, 23.8930, 57.5171,\n",
-      "         4.9486, 24.0208, 13.6062, 44.4289, 31.2610,  4.0193, 12.4009,  5.6658,\n",
-      "        43.0318, 41.9051, 27.0419,  7.6923, 16.9890,  1.3098, 10.9075, 31.5447,\n",
-      "        40.9218, 17.6991,  0.5313, 17.8790, 23.1713, 27.1593,  2.7143,  2.2995,\n",
-      "        43.4573, 24.3312,  9.6550, 12.1840, 29.8788,  2.3538,  7.7584, 14.6252,\n",
-      "         6.2881, 23.5943,  6.0430,  2.3529,  0.8851,  8.8517, 26.0404,  8.4959,\n",
-      "        35.1430, 20.2795], device='cuda:0')\n",
-      "2024-12-11 13:40:57,431 INFO     Batch 5 \n",
-      "Batch:  75%|███████▌  | 6/8 [03:07<01:02, 31.22s/it]2024-12-11 13:40:57,435 INFO     Batch 6 Initial distance: 17.427263259887695 per sample\n",
-      "2024-12-11 13:41:10,819 INFO     iteration: 1, avg. distance 23.71003532409668 per sample\n",
-      "2024-12-11 13:41:28,804 INFO     iteration: 2, avg. distance 17.877588272094727 per sample\n",
-      "2024-12-11 13:41:28,807 INFO     iteration: 2, avg. distance tensor([39.6502, 48.5152, 17.7618, 23.7874, 23.3525,  8.0725,  9.8212,  2.0135,\n",
-      "        12.3639, 22.3919, 21.4216, 25.4651, 15.3634,  0.2495, 27.9673,  3.2518,\n",
-      "        20.0396, 29.3020,  5.0028,  0.1711,  0.5038, 16.3465,  7.1782,  9.9697,\n",
-      "         9.9392, 16.1139, 50.1513, 34.7662, 12.5871,  3.7404,  5.1593, 34.6602,\n",
-      "        31.4391, 17.3484, 16.0570,  9.0646, 37.1402,  9.0535, 18.3022,  8.3219,\n",
-      "        26.7772,  2.2105,  0.1575, 17.9820, 19.4944, 43.4632,  0.6920, 32.8019,\n",
-      "        20.7728, 25.7206], device='cuda:0')\n",
-      "2024-12-11 13:41:28,808 INFO     Batch 6 \n",
-      "Batch:  88%|████████▊ | 7/8 [03:38<00:31, 31.27s/it]2024-12-11 13:41:28,812 INFO     Batch 7 Initial distance: 24.551424026489258 per sample\n",
-      "2024-12-11 13:41:31,510 INFO     iteration: 1, avg. distance 32.30889892578125 per sample\n",
-      "2024-12-11 13:41:35,116 INFO     iteration: 2, avg. distance 18.492856979370117 per sample\n",
-      "2024-12-11 13:41:35,118 INFO     iteration: 2, avg. distance tensor([10.0259, 12.6369,  5.5962, 23.6726, 34.3852, 12.0872, 18.1951, 14.8514,\n",
-      "        29.9312, 23.5470], device='cuda:0')\n",
-      "2024-12-11 13:41:35,119 INFO     Batch 7 \n",
-      "Batch: 100%|██████████| 8/8 [03:45<00:00, 28.14s/it]\n",
-      "2024-12-11 13:41:35,128 INFO     Finished attack: HSJ\n",
-      "2024-12-11 13:41:35,130 INFO     Preparing results for attack: rmia\n",
-      "2024-12-11 13:41:35,131 INFO     Preparing results for attack: population\n",
-      "2024-12-11 13:41:35,132 INFO     Preparing results for attack: lira\n",
-      "2024-12-11 13:41:35,133 INFO     Preparing results for attack: loss_traj\n",
-      "2024-12-11 13:41:35,133 INFO     Preparing results for attack: HSJ\n",
-      "2024-12-11 13:41:35,134 INFO     Auditing completed\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from mia_utils.cifar_handler import CifarInputHandler\n",
     "\n",
@@ -577,23 +265,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2024-12-11 13:41:35,400 INFO     Initializing report handler...\n",
-      "2024-12-11 13:41:35,402 INFO     report_dir set to: ./leakpro_output/results\n",
-      "2024-12-11 13:41:35,403 INFO     Saving results for rmia\n",
-      "2024-12-11 13:41:40,807 INFO     Saving results for population\n",
-      "2024-12-11 13:41:45,987 INFO     Saving results for lira\n",
-      "2024-12-11 13:41:51,253 INFO     Saving results for loss_traj\n",
-      "2024-12-11 13:41:56,788 INFO     Saving results for HSJ\n"
+      "2024-12-18 01:00:10,883 INFO     Initializing report handler...\n",
+      "2024-12-18 01:00:10,885 INFO     report_dir set to: ./leakpro_output/results\n"
      ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<Figure size 640x480 with 0 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
     }
    ],
    "source": [
@@ -621,7 +295,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "id": "1d91c7e0",
    "metadata": {},
    "outputs": [
@@ -629,7 +303,11 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2024-12-11 13:45:59,879 INFO     PDF compiled\n"
+      "2024-12-18 01:00:22,740 INFO     No results of type GIAResults found.\n",
+      "2024-12-18 01:00:22,743 INFO     No results of type SinglingOutResults found.\n",
+      "2024-12-18 01:00:22,744 INFO     No results of type InferenceResults found.\n",
+      "2024-12-18 01:00:22,745 INFO     No results of type LinkabilityResults found.\n",
+      "2024-12-18 01:00:31,133 INFO     PDF compiled\n"
      ]
     },
     {
@@ -640,38 +318,34 @@
      },
      "metadata": {},
      "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<Figure size 1100x400 with 0 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<Figure size 1100x700 with 0 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
+    }
+   ],
+   "source": [
+    "# Simply create a report by collecting and compiling latex text\n",
+    "report_handler.create_report()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "553f64d5",
+   "metadata": {},
+   "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "<Figure size 1100x700 with 0 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-12-18 01:00:42,989 INFO     No results of type GIAResults found.\n",
+      "2024-12-18 01:00:42,992 INFO     No results of type SinglingOutResults found.\n",
+      "2024-12-18 01:00:42,993 INFO     No results of type InferenceResults found.\n",
+      "2024-12-18 01:00:42,994 INFO     No results of type LinkabilityResults found.\n",
+      "2024-12-18 01:00:51,396 INFO     PDF compiled\n"
+     ]
     },
     {
      "data": {
       "text/plain": [
-       "<Figure size 1100x400 with 0 Axes>"
+       "<Figure size 640x480 with 0 Axes>"
       ]
      },
      "metadata": {},
@@ -679,10 +353,15 @@
     }
    ],
    "source": [
+    "#### Have more granular control\n",
+    "\n",
     "# Use the ReportHandler and load all the saved results\n",
     "report_handler.load_results()\n",
     "\n",
-    "# Create results and collect corresponding latex texts\n",
+    "# Create results and collect corresponding latex texts. Use different methods to create results for a specific type\n",
+    "# report_handler.create_results_mia()\n",
+    "# report_handler.create_results_gia()\n",
+    "# report_handler.create_results_syn()\n",
     "report_handler.create_results_all()\n",
     "\n",
     "# Create the report by compiling the latex text\n",
diff --git a/leakpro/metrics/attack_result.py b/leakpro/metrics/attack_result.py
index 2781638..cdf2cb3 100755
--- a/leakpro/metrics/attack_result.py
+++ b/leakpro/metrics/attack_result.py
@@ -286,7 +286,7 @@ def save(self:Self, path: str, name: str, config:dict = None, show_plot:bool = F
             "tpr": self.tpr.tolist(),
             "fpr": self.fpr.tolist(),
             "roc_auc": self.roc_auc,
-            "config": config,
+            "config": result_config,
             "fixed_fpr": fixed_fpr_table,
             "audit_indices": self.audit_indices.tolist() if self.audit_indices is not None else None,
             "signal_values": self.signal_values.tolist() if self.signal_values is not None else None,
@@ -310,7 +310,7 @@ def save(self:Self, path: str, name: str, config:dict = None, show_plot:bool = F
         temp_res.id = self.id
         self.create_plot(results = [temp_res],
                         save_dir = save_path,
-                        save_name = "ROC",
+                        save_name = name,
                         show_plot = show_plot
                         )
 
@@ -415,7 +415,7 @@ def create_plot(
 
         plt.xlabel("False positive rate (FPR)")
         plt.ylabel("True positive rate (TPR)")
-        plt.title(save_name+"ROC Curve")
+        plt.title(save_name+" ROC Curve")
         plt.savefig(fname=f"{filename}.png", dpi=1000, bbox_inches="tight")
 
         if show_plot:
@@ -474,18 +474,16 @@ def create_results(
     @staticmethod
     def _latex(
             results: list,
-            save_dir: str,
+            save_dir: str, # noqa: ARG004
             save_name: str
         ) -> str:
         """Latex method for MIAResult."""
 
-        filename = f"{save_dir}/{save_name}"
-
         # Input mia results image
         latex_content = f"""
         \\subsection{{{" ".join(save_name.split("_"))}}}
         \\begin{{figure}}[ht]
-        \\includegraphics[width=0.8\\textwidth]{{{filename}.png}}
+        \\includegraphics[width=0.8\\textwidth]{{{save_name}.png}}
         \\end{{figure}}
         """
 
@@ -493,9 +491,7 @@ def _latex(
         latex_content += """
         \\resizebox{\\linewidth}{!}{%
         \\begin{tabularx}{\\textwidth}{l c l l l l}
-        Attack name & attack config & TPR: 1.0\\%FPR & 0.1\\%FPR & 0.01\\%FPR & 0.0\\%FPR \\\\
-        \\hline
-        """
+        Attack name & attack config & TPR: 1.0\\%FPR & 0.1\\%FPR & 0.01\\%FPR & 0.0\\%FPR \\\\ \\hline """ # noqa: W291
 
         # Convert config to latex table input
         def config_latex_style(config: str) -> str:
@@ -505,10 +501,9 @@ def config_latex_style(config: str) -> str:
 
         # Append all mia results to table
         for res in results:
-            config = config_latex_style(res.id)
-            latex_content += f"""{"-".join(res.resultname.split("_"))} & {config} & {res.fixed_fpr_table["TPR@1.0%FPR"]} &
-                                {res.fixed_fpr_table["TPR@0.1%FPR"]} & {res.fixed_fpr_table["TPR@0.01%FPR"]} &
-                                {res.fixed_fpr_table["TPR@0.0%FPR"]} \\\\ \\hline"""
+            config = config_latex_style(get_config_name(res.config))
+            latex_content += f"""
+            {"-".join(res.resultname.split("_"))} & {config} & {res.fixed_fpr_table["TPR@1.0%FPR"]} & {res.fixed_fpr_table["TPR@0.1%FPR"]} & {res.fixed_fpr_table["TPR@0.01%FPR"]} & {res.fixed_fpr_table["TPR@0.0%FPR"]} \\\\ \\hline """ # noqa: E501
         latex_content += """
         \\end{tabularx}
         }
diff --git a/leakpro/reporting/report_handler.py b/leakpro/reporting/report_handler.py
index 2b9f39e..781d120 100644
--- a/leakpro/reporting/report_handler.py
+++ b/leakpro/reporting/report_handler.py
@@ -156,6 +156,12 @@ def create_results_syn(
     def create_report(self:Self) -> None:
         """Method to create PDF report."""
 
+        # Make sure results have been read and created
+        if not hasattr(self, "results"):
+            self.load_results()
+        if self.pdf_results and all(not value for value in self.pdf_results.values()):
+            self.create_results_all()
+
         # Create initial part of the document.
         self._init_pdf()
 
@@ -169,6 +175,9 @@ def create_report(self:Self) -> None:
         # Compile the PDF
         self._compile_pdf()
 
+        # Empty result variables
+        self._reset_result()
+
     def _init_pdf(self:Self) -> None:
         self.latex_content = """
         \\documentclass{article}
@@ -202,3 +211,8 @@ def _compile_pdf(self:Self) -> None:
         except Exception as e:
             self.logger.info(f"Could not compile PDF: {e}")
             self.logger.info("Make sure to install pdflatex with apt install texlive-latex-base")
+
+    def _reset_result(self:Self) -> None:
+        self.results = []
+        for key in self.pdf_results:
+            self.pdf_results[key] = []
diff --git a/leakpro/tests/test_attack_result/test_attack_result.py b/leakpro/tests/test_attack_result/test_attack_result.py
index df5af75..7b74a18 100644
--- a/leakpro/tests/test_attack_result/test_attack_result.py
+++ b/leakpro/tests/test_attack_result/test_attack_result.py
@@ -99,7 +99,7 @@ def test_save_load_miaresult(self:Self) -> None:
 
         assert os.path.isdir(save_path)
         assert os.path.exists(f"{save_path}/data.json")
-        assert os.path.exists(f"{save_path}/ROC.png")
+        assert os.path.exists(f"{save_path}/{name}.png")
         assert os.path.exists(f"{save_path}/SignalHistogram.png")
 
         # Test loading
@@ -130,18 +130,19 @@ def test_latex(self:Self, mocker: MockerFixture) -> None:
         """Test if the LaTeX content is generated correctly."""
 
         result = [mocker.Mock(id="attack-config-1", resultname="test_attack_1",\
-                     fixed_fpr_table={"TPR@1.0%FPR": 0.90, "TPR@0.1%FPR": 0.80, "TPR@0.01%FPR": 0.70, "TPR@0.0%FPR": 0.60})]
+                     fixed_fpr_table={"TPR@1.0%FPR": 0.90, "TPR@0.1%FPR": 0.80, "TPR@0.01%FPR": 0.70, "TPR@0.0%FPR": 0.60},
+                     config={"training_data_fraction": 0.5, "num_shadow_models": 3, "online": True})]
 
-        subsection = "attack_comparison"
-        filename = f"{self.temp_dir}/{subsection}"
+        name = "attack_comparison"
+        filename = f"{self.temp_dir}/{name}"
+
+        latex_content = MIAResult()._latex(result, save_dir=self.temp_dir, save_name=name)
 
-        latex_content = MIAResult()._latex(result, save_dir=self.temp_dir, save_name=subsection)
-        print(latex_content)
         # Check that the subsection is correctly included
         assert "\\subsection{attack comparison}" in latex_content
 
         # Check that the figure is correctly included
-        assert f"\\includegraphics[width=0.8\\textwidth]{{{filename}.png}}" in latex_content
+        assert f"\\includegraphics[width=0.8\\textwidth]{{{name}.png}}" in latex_content
 
         # Check that the table header is correct
         assert "Attack name & attack config & TPR: 1.0\\%FPR & 0.1\\%FPR & 0.01\\%FPR & 0.0\\%FPR" in latex_content