Submissions/ML-Submission3/ML_Submission3.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j8cvJhZS7UB2"
      },
      "source": [
        "# Image Classification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "63zxPYZL7DbL"
      },
      "outputs": [],
      "source": [
        "# Essential packages\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "\n",
        "# Torch packages\n",
        "#!pip install torch\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "torch.set_float32_matmul_precision('medium')\n",
        "\n",
        "# Torch Metrics\n",
        "#!pip install torchmetrics\n",
        "import torchmetrics\n",
        "\n",
        "# Torch Vision\n",
        "#!pip install torchvision\n",
        "from torchvision import transforms\n",
        "\n",
        "# Lightning\n",
        "#!pip install pytorch-lightning\n",
        "import pytorch_lightning as pl\n",
        "from pytorch_lightning.loggers import TensorBoardLogger\n",
        "\n",
        "# Optuna\n",
        "#!pip install optuna\n",
        "#!pip install optuna-integration[pytorch_lightning]\n",
        "import optuna\n",
        "from optuna.integration.pytorch_lightning import PyTorchLightningPruningCallback\n",
        "\n",
        "# Scikit-Learn packages\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.utils import class_weight\n",
        "from sklearn.metrics import (\n",
        "    roc_curve, auc, precision_recall_curve, roc_auc_score,\n",
        "    precision_score, recall_score, f1_score, classification_report\n",
        ")\n",
        "\n",
        "# Load data\n",
        "X_train = np.load(\"./data/Xtrain1.npy\")\n",
        "y_train = np.load(\"./data/Ytrain1.npy\")\n",
        "X_train_extra = np.load(\"./data/Xtrain1_extra.npy\")\n",
        "X_test = np.load(\"./data/Xtest1.npy\")\n",
        "\n",
        "print(f\"X_train shape: {X_train.shape}\")\n",
        "print(f\"y_train shape: {y_train.shape}\")\n",
        "print(f\"X_train_extra shape: {X_train_extra.shape}\")\n",
        "print(f\"X_test shape: {X_test.shape}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w9nV0Gge8nd_"
      },
      "source": [
        "## Data Exploration"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Print images with matplotlib"
      ],
      "metadata": {
        "id": "2xW0bOvs6FoP"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cOJgdKdm8n2W"
      },
      "outputs": [],
      "source": [
        "print(f\"Number of images: {X_train.shape[0]}, Image size: {X_train.shape[1]}\")\n",
        "print(f\"Number of classes: {len(np.unique(y_train))}\\n\")\n",
        "\n",
        "# Print a specific image\n",
        "#img1 = X_train[1,:].reshape((48,48))\n",
        "#imgplot = plt.imshow(img1, cmap='gray')\n",
        "\n",
        "# Print all images\n",
        "num_images = X_train.shape[0]\n",
        "img_shape = (48, 48)\n",
        "\n",
        "images_per_figure = 200\n",
        "cols = 20\n",
        "rows = int(np.ceil(images_per_figure / cols))\n",
        "\n",
        "# Loop through all images in batches\n",
        "for start_idx in range(0, num_images, images_per_figure):\n",
        "    fig, axes = plt.subplots(rows, cols, figsize=(cols * 2, rows * 2))\n",
        "    axes = axes.flatten()\n",
        "\n",
        "    for i, ax in enumerate(axes):\n",
        "        img_idx = start_idx + i\n",
        "        if img_idx < num_images:\n",
        "            img = X_train[img_idx, :].reshape(img_shape)\n",
        "            ax.imshow(img, cmap='gray')\n",
        "            ax.axis('off')\n",
        "        else:\n",
        "            ax.axis('off')\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save images to PDF file"
      ],
      "metadata": {
        "id": "0mO0Ss7o6KVW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install necessary packages\n",
        "!pip install PyPDF2 pillow\n",
        "\n",
        "from PyPDF2 import PdfMerger\n",
        "from PIL import Image\n",
        "import os\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "X = np.array(X_train)  # Training images\n",
        "y = np.array(y_train)  # Corresponding labels\n",
        "\n",
        "num_images = X.shape[0]\n",
        "img_shape = (48, 48)\n",
        "cols = 10\n",
        "images_per_batch = 100\n",
        "rows = int(np.ceil(images_per_batch / cols))\n",
        "batch_image_paths = []\n",
        "\n",
        "for batch_start in range(0, num_images, images_per_batch):\n",
        "    fig, axes = plt.subplots(rows, cols, figsize=(cols * 2, rows * 2))\n",
        "    axes = axes.flatten()\n",
        "\n",
        "    for i, ax in enumerate(axes):\n",
        "        img_idx = batch_start + i\n",
        "        if img_idx < num_images:\n",
        "            img = X[img_idx, :].reshape(img_shape)\n",
        "            ax.imshow(img, cmap='gray')\n",
        "            ax.set_title(f'ID: {img_idx}, [{y[img_idx]}]')\n",
        "            ax.axis('off')\n",
        "        else:\n",
        "            ax.axis('off')\n",
        "\n",
        "    batch_filename = f'batch_{batch_start // images_per_batch}.png'\n",
        "    plt.savefig(batch_filename, bbox_inches='tight', pad_inches=0.1, dpi=120)\n",
        "    batch_image_paths.append(batch_filename)\n",
        "    plt.close(fig)\n",
        "\n",
        "batch_pdfs = []\n",
        "for batch_image in batch_image_paths:\n",
        "    img = Image.open(batch_image)\n",
        "    if img.mode != 'RGB':\n",
        "        img = img.convert('RGB')\n",
        "    img_resized = img.resize((img.width // 2, img.height // 2), Image.Resampling.LANCZOS)\n",
        "    pdf_filename = batch_image.replace('.png', '.pdf')\n",
        "    img_resized.save(pdf_filename, 'PDF', resolution=100.0, quality=100, optimize=True)\n",
        "    batch_pdfs.append(pdf_filename)\n",
        "\n",
        "pdf_merger = PdfMerger()\n",
        "for pdf_file in batch_pdfs:\n",
        "    pdf_merger.append(pdf_file)\n",
        "\n",
        "final_pdf_filename = 'all_images_merged.pdf'\n",
        "pdf_merger.write(final_pdf_filename)\n",
        "pdf_merger.close()\n",
        "\n",
        "print(f\"Final merged PDF saved as '{final_pdf_filename}'\")\n",
        "\n",
        "for batch_image in batch_image_paths:\n",
        "    os.remove(batch_image)\n",
        "for pdf_file in batch_pdfs:\n",
        "    os.remove(pdf_file)\n",
        "\n",
        "print(\"Intermediate batch images and PDF files deleted.\")"
      ],
      "metadata": {
        "id": "qz8LpI94mAyt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## CNN Model Training"
      ],
      "metadata": {
        "id": "TJ6C5aKZ0Coi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Preprocessing Data"
      ],
      "metadata": {
        "id": "jiIp54o6UjcU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install imblearn\n",
        "from imblearn.over_sampling import SMOTE\n",
        "\n",
        "# Reshape the images to (num_samples, 48*48) for SMOTE (flatten images)\n",
        "X_train_reshaped = X_train.reshape(-1, 48, 48)\n",
        "X_train_reshaped = np.expand_dims(X_train_reshaped, axis=1)\n",
        "\n",
        "num_samples, channels, height, width = X_train_reshaped.shape\n",
        "X_train_flat = X_train.reshape(num_samples, -1)  # Shape: (num_samples, 48*48)\n",
        "\n",
        "print(f\"Class distribution before SMOTE: {np.bincount(y_train)}\")\n",
        "\n",
        "# Split into train and validation sets (no SMOTE applied to validation data)\n",
        "X_train_split, X_val_split, y_train_split, y_val_split = train_test_split(\n",
        "    X_train_flat, y_train, test_size=0.20, random_state=1337\n",
        ")\n",
        "\n",
        "# Apply SMOTE to oversample only class 0 in the training split\n",
        "smote = SMOTE(sampling_strategy={0: int(y_train_split[y_train_split == 1].shape[0])})  # Balance to the number of class 1\n",
        "X_resampled, y_resampled = smote.fit_resample(X_train_split, y_train_split)\n",
        "\n",
        "print(f\"Class distribution after SMOTE: {np.bincount(y_resampled)}\\n\")\n",
        "\n",
        "# Separate synthetic samples from original samples\n",
        "num_original_samples = len(X_train_split)\n",
        "num_resampled_samples = len(X_resampled)\n",
        "\n",
        "# Find the synthetic samples\n",
        "synthetic_samples = X_resampled[num_original_samples:]\n",
        "synthetic_labels = y_resampled[num_original_samples:]\n",
        "\n",
        "# Reshape the resampled data back to (num_samples, 1, 48, 48)\n",
        "synthetic_samples_images = synthetic_samples.reshape(-1, channels, height, width)\n",
        "\n",
        "# Visualize some oversampled images\n",
        "fig, axes = plt.subplots(2, 5, figsize=(10, 5))\n",
        "axes = axes.ravel()\n",
        "\n",
        "for i in range(10):\n",
        "    axes[i].imshow(synthetic_samples_images[i, 0], cmap='gray')\n",
        "    axes[i].set_title(f'Class: {synthetic_labels[i]}')\n",
        "    axes[i].axis('off')\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "onh2MFLTUHZo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torchvision import transforms\n",
        "\n",
        "# Custom dataset class\n",
        "class CraterDataset(Dataset):\n",
        "    def __init__(self, images, labels, transform=None):\n",
        "        self.images = images\n",
        "        self.labels = labels\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.images)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        image = self.images[idx]\n",
        "        label = self.labels[idx]\n",
        "\n",
        "        # Convert numpy image to PIL image for augmentation\n",
        "        image_pil = Image.fromarray(image.squeeze(), mode='L')  # 'L' for grayscale\n",
        "\n",
        "        # Apply transformations (if any)\n",
        "        image = self.transform(image_pil)\n",
        "\n",
        "        return image, torch.tensor(label, dtype=torch.float32)\n",
        "\n",
        "# Data transformations (augmentation)\n",
        "train_transform = transforms.Compose([\n",
        "    #transforms.RandomHorizontalFlip(),\n",
        "    #transforms.RandomVerticalFlip(),\n",
        "    #transforms.RandomRotation(90),\n",
        "    transforms.ToTensor(),  # Convert the PIL image to a tensor\n",
        "])\n",
        "\n",
        "# Validation transform (just conversion to tensor)\n",
        "val_transform = transforms.Compose([\n",
        "    transforms.ToTensor(),  # Convert the PIL image to a tensor\n",
        "])\n",
        "\n",
        "# Reshape the resampled training data and validation data back to (num_samples, 1, 48, 48)\n",
        "X_train_reshaped = X_resampled.reshape(-1, 1, 48, 48)\n",
        "X_val_reshaped = X_val_split.reshape(-1, 1, 48, 48)\n",
        "X_train_extra_reshaped = X_train_extra.reshape(-1, 1, 48, 48)\n",
        "\n",
        "# Check shapes\n",
        "print(\"X_train_reshaped shape:\", X_train_reshaped.shape)\n",
        "print(\"X_val_reshaped shape:\", X_val_reshaped.shape)\n",
        "print(\"X_train_extra_reshaped shape:\", X_train_extra_reshaped.shape)\n",
        "\n",
        "# Create datasets with transforms\n",
        "train_dataset = CraterDataset(X_train_reshaped, y_resampled, transform=train_transform)\n",
        "val_dataset = CraterDataset(X_val_reshaped, y_val_split, transform=val_transform)\n",
        "\n",
        "# Create data loaders\n",
        "batch_size = 32\n",
        "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n",
        "\n",
        "# Calculate class weights to handle imbalance in the resampled training data\n",
        "class_weights = class_weight.compute_class_weight(\n",
        "    class_weight='balanced',\n",
        "    classes=np.unique(y_resampled),\n",
        "    y=y_resampled\n",
        ")\n",
        "class_weights_tensor = torch.tensor(class_weights, dtype=torch.float32)\n",
        "print(class_weights_tensor)"
      ],
      "metadata": {
        "id": "etAyUf_g0CN2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### CNN Model Training"
      ],
      "metadata": {
        "id": "sYq3ZF6YUkvX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the model\n",
        "class NewCNN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(NewCNN, self).__init__()\n",
        "\n",
        "        # First block: 1 -> 32 filters\n",
        "        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1)\n",
        "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n",
        "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, padding=1)\n",
        "        self.bn1 = nn.BatchNorm2d(64)\n",
        "\n",
        "        # Second block: 64 -> 128 filters\n",
        "        self.conv4 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n",
        "        self.conv5 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n",
        "        self.conv6 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n",
        "        self.bn2 = nn.BatchNorm2d(256)\n",
        "\n",
        "        # Third block: 256 -> 512 filters\n",
        "        self.conv7 = nn.Conv2d(256, 512, kernel_size=3, padding=1)\n",
        "        self.bn3 = nn.BatchNorm2d(512)\n",
        "\n",
        "        # Global Average Pooling\n",
        "        self.gap = nn.AdaptiveAvgPool2d(1)\n",
        "\n",
        "        # Fully connected layers\n",
        "        self.fc1 = nn.Linear(512, 256)\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        self.fc2 = nn.Linear(256, 1)  # Binary classification output\n",
        "\n",
        "        # Activation function\n",
        "        self.activation = nn.LeakyReLU(0.05)\n",
        "\n",
        "    def forward(self, x):\n",
        "        # First block: Conv + BN + Activation + Pool\n",
        "        x = self.activation(self.conv1(x))\n",
        "        x = self.activation(self.conv2(x))\n",
        "        x = self.activation(self.conv3(x))\n",
        "        x = self.bn1(x)\n",
        "        x = nn.MaxPool2d(2, 2)(x)\n",
        "\n",
        "        # Second block: Conv + BN + Activation + Pool\n",
        "        x = self.activation(self.conv4(x))\n",
        "        x = self.activation(self.conv5(x))\n",
        "        x = self.activation(self.conv6(x))\n",
        "        x = self.bn2(x)\n",
        "        x = nn.MaxPool2d(2, 2)(x)\n",
        "\n",
        "        # Third block: Conv + BN + Activation + Pool\n",
        "        x = self.activation(self.conv7(x))\n",
        "        x = self.bn3(x)\n",
        "        x = nn.MaxPool2d(2, 2)(x)\n",
        "\n",
        "        # Global Average Pooling\n",
        "        x = self.gap(x)\n",
        "        x = x.view(x.size(0), -1)  # Flatten for fully connected layer\n",
        "\n",
        "        # Fully Connected Layers\n",
        "        x = self.activation(self.fc1(x))\n",
        "        x = self.dropout(x)\n",
        "\n",
        "        # Output layer\n",
        "        return self.fc2(x)"
      ],
      "metadata": {
        "id": "zR7EoTnfXvcL"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torchmetrics\n",
        "\n",
        "class ClassificationTask(pl.LightningModule):\n",
        "    def __init__(self, model=None, criterion=None, threshold=0.5, optimizer_type='Adam', lr=1e-5, scheduler_type=None):\n",
        "        super(ClassificationTask, self).__init__()\n",
        "\n",
        "        self.model = model\n",
        "        self.criterion = criterion\n",
        "        self.lr = lr\n",
        "        self.optimizer_type = optimizer_type\n",
        "        self.scheduler_type = scheduler_type\n",
        "        self.threshold = threshold\n",
        "\n",
        "        # Define metrics for train, validation, and test\n",
        "        self.metrics = {\n",
        "            'train': {\n",
        "                'acc': torchmetrics.Accuracy(task=\"binary\", threshold=self.threshold),\n",
        "                'f1': torchmetrics.F1Score(task=\"multiclass\", num_classes=2, threshold=self.threshold, average=\"macro\"),\n",
        "                'losses': []\n",
        "            },\n",
        "            'val': {\n",
        "                'acc': torchmetrics.Accuracy(task=\"binary\", threshold=self.threshold),\n",
        "                'f1': torchmetrics.F1Score(task=\"multiclass\", num_classes=2, threshold=self.threshold, average=\"macro\"),\n",
        "                'losses': []\n",
        "            },\n",
        "            'test': {\n",
        "                'acc': torchmetrics.Accuracy(task=\"binary\", threshold=self.threshold),\n",
        "                'f1': torchmetrics.F1Score(task=\"multiclass\", num_classes=2, threshold=self.threshold, average=\"macro\"),\n",
        "                'losses': []\n",
        "            }\n",
        "        }\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.model(x)\n",
        "\n",
        "    def _step(self, batch, phase):\n",
        "        x, y = batch\n",
        "        logits = self(x).view(-1).to(self.device)  # Forward pass\n",
        "\n",
        "        # Ensure metrics are also on the same device\n",
        "        self.metrics[phase]['acc'].to(self.device)\n",
        "        self.metrics[phase]['f1'].to(self.device)\n",
        "\n",
        "        preds = torch.sigmoid(logits) >= self.threshold\n",
        "        loss = self.criterion(logits, y)\n",
        "\n",
        "        # Update the metrics\n",
        "        self.metrics[phase]['losses'].append(loss.item())\n",
        "        self.metrics[phase]['acc'](preds, y)\n",
        "        self.metrics[phase]['f1'](preds, y)\n",
        "\n",
        "        return loss\n",
        "\n",
        "    def training_step(self, batch, batch_idx):\n",
        "        return self._step(batch, 'train')\n",
        "\n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        return self._step(batch, 'val')\n",
        "\n",
        "    def test_step(self, batch, batch_idx):\n",
        "        return self._step(batch, 'test')\n",
        "\n",
        "    def _epoch_end(self, phase):\n",
        "        avg_loss = torch.tensor(self.metrics[phase]['losses']).mean()\n",
        "        avg_acc = self.metrics[phase]['acc'].compute()\n",
        "        avg_f1 = self.metrics[phase]['f1'].compute()\n",
        "\n",
        "        # Log the metrics\n",
        "        self.log(f'{phase}_loss_epoch', avg_loss, prog_bar=True)\n",
        "        self.log(f'{phase}_acc_epoch', avg_acc, prog_bar=True)\n",
        "        self.log(f'{phase}_f1_epoch', avg_f1, prog_bar=True)\n",
        "\n",
        "        # Reset metrics for the next epoch\n",
        "        self.metrics[phase]['losses'].clear()\n",
        "        self.metrics[phase]['acc'].reset()\n",
        "        self.metrics[phase]['f1'].reset()\n",
        "\n",
        "    def on_train_epoch_end(self):\n",
        "        self._epoch_end('train')\n",
        "\n",
        "    def on_validation_epoch_end(self):\n",
        "        self._epoch_end('val')\n",
        "\n",
        "    def on_test_epoch_end(self):\n",
        "        self._epoch_end('test')\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        if self.optimizer_type == 'Adam':\n",
        "            optimizer = torch.optim.AdamW(self.model.parameters(), lr=self.lr, weight_decay=1e-5)\n",
        "        else:\n",
        "            raise ValueError(f\"Unsupported optimizer type: {self.optimizer_type}\")\n",
        "\n",
        "        if self.scheduler_type == 'ReduceLROnPlateau':\n",
        "            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=5)\n",
        "            return {'optimizer': optimizer, 'lr_scheduler': {'scheduler': scheduler, 'monitor': 'val_f1_epoch'}}\n",
        "        elif self.scheduler_type == 'StepLR':\n",
        "            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n",
        "            return {'optimizer': optimizer, 'lr_scheduler': {'scheduler': scheduler}}\n",
        "        else:\n",
        "            return optimizer"
      ],
      "metadata": {
        "id": "roVEEaz8H1qj"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "optuna.logging.set_verbosity(optuna.logging.WARNING)\n",
        "\n",
        "# This redefinition is just a bug fix...\n",
        "class OptunaPruning(PyTorchLightningPruningCallback, pl.Callback):\n",
        "    def __init__(self, *args, **kwargs):\n",
        "        super().__init__(*args, **kwargs)\n",
        "\n",
        "# Define Optuna's objective function, optimized for F1-score\n",
        "def Objective(train_loader, val_loader, class_weights_tensor, trial):\n",
        "    # Suggest hyperparameters to tune\n",
        "    lr = trial.suggest_float('lr', 1e-7, 1e-3, log=True)\n",
        "    dropout_rate = trial.suggest_float('dropout_rate', 0.1, 0.7)\n",
        "    optimizer_type = trial.suggest_categorical('optimizer_type', ['Adam'])\n",
        "    threshold_opt = trial.suggest_float('threshold', 0.1, 0.9)\n",
        "\n",
        "    # Modify the CNN model with the trial's suggested dropout rate\n",
        "    model = NewCNN()\n",
        "    model.dropout = nn.Dropout(dropout_rate)\n",
        "\n",
        "    # Define the loss function with class weights\n",
        "    criterion = nn.BCEWithLogitsLoss(pos_weight=class_weights_tensor[1])\n",
        "\n",
        "    # Create a classification task instance using suggested hyperparameters\n",
        "    classification_task = ClassificationTask(\n",
        "        model=model,\n",
        "        criterion=criterion,\n",
        "        threshold=threshold_opt,\n",
        "        optimizer_type=optimizer_type,\n",
        "        lr=lr,\n",
        "        scheduler_type='ReduceLROnPlateau'\n",
        "    )\n",
        "\n",
        "    # Early stopping callback based on F1 score\n",
        "    early_stopping = pl.callbacks.EarlyStopping(\n",
        "        monitor='val_f1_epoch',\n",
        "        patience=10,\n",
        "        verbose=True,\n",
        "        mode='max'\n",
        "    )\n",
        "\n",
        "    # Model checkpoint to save the best model's weights\n",
        "    checkpoint_callback = pl.callbacks.ModelCheckpoint(\n",
        "        dirpath=f\"lightning_logs/optuna_trials/trial_{trial.number}\",\n",
        "        filename=f\"best_model_trial_{trial.number}\",\n",
        "        monitor='val_f1_epoch',\n",
        "        save_top_k=1,\n",
        "        mode='max'\n",
        "    )\n",
        "\n",
        "    # Set up the logger to store logs in the optuna_trials directory\n",
        "    logger = pl.loggers.TensorBoardLogger(\"lightning_logs/optuna_trials\", name=f\"trial_{trial.number}\")\n",
        "\n",
        "    # Set up the PyTorch Lightning Trainer with pruning callback\n",
        "    trainer = pl.Trainer(\n",
        "        max_epochs=100,\n",
        "        logger=logger,\n",
        "        accelerator=\"auto\",\n",
        "        strategy=\"auto\",\n",
        "        devices=\"auto\",\n",
        "        callbacks=[\n",
        "            early_stopping,\n",
        "            checkpoint_callback,\n",
        "            OptunaPruning(trial, monitor=\"val_f1_epoch\")\n",
        "        ],\n",
        "        #log_every_n_steps=0,        # No logging during steps\n",
        "        #enable_progress_bar=False,  # Disable the progress bar\n",
        "        #enable_model_summary=False  # Disable the model summary output\n",
        "    )\n",
        "\n",
        "    # Train the model\n",
        "    trainer.fit(classification_task, train_loader, val_loader)\n",
        "\n",
        "    # Optuna will still compute and optimize based on the F1 score on the validation set\n",
        "    f1_metric = torchmetrics.F1Score(task=\"multiclass\", num_classes=2, threshold=threshold_opt, average=\"macro\")\n",
        "\n",
        "    classification_task.eval()  # Switch to evaluation mode\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for batch in val_loader:\n",
        "            x, y = batch\n",
        "\n",
        "            logits = classification_task.model(x).flatten()\n",
        "            preds = torch.sigmoid(logits) >= threshold_opt\n",
        "\n",
        "            f1_metric.update(preds, y)\n",
        "\n",
        "    # Compute the weighted F1 score\n",
        "    macro_f1_score = f1_metric.compute().item()\n",
        "\n",
        "    # Print the F1 score for the current trial\n",
        "    print(f\"Trial {trial.number}: Macro F1 Score = {macro_f1_score:.6f}\")\n",
        "\n",
        "    # Return the macro F1 score\n",
        "    return macro_f1_score"
      ],
      "metadata": {
        "id": "HiOWGhKL5XCQ"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import logging\n",
        "#log = logging.getLogger(\"pytorch_lightning\")\n",
        "#log.propagate = False\n",
        "#log.setLevel(logging.ERROR)\n",
        "\n",
        "# Set up the Optuna study and optimize the objective\n",
        "study = optuna.create_study(direction=\"maximize\")\n",
        "study.optimize(\n",
        "    lambda trial: Objective(train_loader, val_loader, class_weights_tensor, trial),\n",
        "    n_trials=50 # Perform N trials of hyperparameter search\n",
        ")\n",
        "\n",
        "# Print the best hyperparameters found by Optuna\n",
        "best_trial = study.best_trial\n",
        "print(\"Best trial:\")\n",
        "print(f\"  F1 score: {best_trial.value}\")\n",
        "print(f\"  Params: \")\n",
        "for key, value in best_trial.params.items():\n",
        "    print(f\"    {key}: {value}\")"
      ],
      "metadata": {
        "id": "vGZkplGhcJWL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load best model and predict"
      ],
      "metadata": {
        "id": "1jMSJm45b-eZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the best model checkpoint based on the best trial number\n",
        "#best_trial_number = best_trial.number\n",
        "#best_model_checkpoint_path = f\"lightning_logs/optuna_trials/trial_{best_trial_number}/best_model_trial_{best_trial_number}.ckpt\"\n",
        "\n",
        "best_model_checkpoint_path = f\"./best_model_trial_2.ckpt\"\n",
        "print(f\"Loading the best model from: {best_model_checkpoint_path}\")\n",
        "\n",
        "# Load the model from the checkpoint\n",
        "task = ClassificationTask.load_from_checkpoint(\n",
        "    checkpoint_path=best_model_checkpoint_path,\n",
        "    criterion=nn.BCEWithLogitsLoss(pos_weight=class_weights_tensor[1]),\n",
        "    model=NewCNN()\n",
        ")\n",
        "task.eval()\n",
        "\n",
        "# Make predictions and get logits (for ROC and AUC)\n",
        "logits_list = []\n",
        "predictions = []\n",
        "with torch.no_grad():\n",
        "    for batch in val_loader:\n",
        "        images, _ = batch\n",
        "        images = images.to(task.device)\n",
        "        logits = task.model(images)\n",
        "        logits_list.append(logits.cpu())\n",
        "\n",
        "# Flatten the logits tensor and apply sigmoid to get probabilities\n",
        "logits = torch.cat(logits_list).flatten().numpy()\n",
        "probabilities = torch.sigmoid(torch.tensor(logits)).numpy()\n",
        "\n",
        "# Convert y_test_split to a tensor and numpy array\n",
        "y_test_tensor = torch.tensor(y_val_split, dtype=torch.int).numpy()\n",
        "\n",
        "# ---- ROC Curve and Youden's J Statistic ----\n",
        "fpr, tpr, thresholds = roc_curve(y_test_tensor, probabilities)\n",
        "roc_auc = auc(fpr, tpr)\n",
        "\n",
        "# Calculate Youden's J statistic for each threshold\n",
        "youden_j = tpr - fpr\n",
        "best_threshold_index = np.argmax(youden_j)\n",
        "best_threshold1 = thresholds[best_threshold_index]\n",
        "print(f'Best threshold according to Youden\\'s J: {best_threshold1:.4f}')\n",
        "\n",
        "#final_predictions = (probabilities >= best_threshold1).astype(int)\n",
        "\n",
        "# ---- ROC Curve ----\n",
        "plt.figure(figsize=(8, 6))\n",
        "plt.plot(fpr, tpr, color='blue', label=f'ROC curve (AUC = {roc_auc:.2f})')\n",
        "plt.plot([0, 1], [0, 1], linestyle='--', color='gray') # Diagonal\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
        "plt.legend(loc=\"lower right\")\n",
        "plt.grid(True)\n",
        "plt.show()\n",
        "\n",
        "# ---- Precision-Recall Curve ----\n",
        "\n",
        "# Calculate the precision-recall curve\n",
        "precision, recall, thresholds = precision_recall_curve(y_test_tensor, probabilities)\n",
        "\n",
        "f1_scores = 2 * (precision * recall) / (precision + recall)\n",
        "\n",
        "best_threshold_index = np.argmax(f1_scores)\n",
        "best_threshold2 = thresholds[best_threshold_index]\n",
        "print(f'Best threshold for maximizing F1 score: {best_threshold2}')\n",
        "\n",
        "#best_trial.params['threshold']\n",
        "final_predictions = (probabilities >= 0.8500936271227173).astype(int)\n",
        "\n",
        "plt.figure(figsize=(8, 6))\n",
        "plt.plot(recall, precision, color='green')\n",
        "plt.xlabel('Recall')\n",
        "plt.ylabel('Precision')\n",
        "plt.title('Precision-Recall Curve')\n",
        "plt.grid(True)\n",
        "plt.show()\n",
        "\n",
        "# ---- Confusion Matrix ----\n",
        "confmat = torchmetrics.ConfusionMatrix(task=\"binary\")\n",
        "confmat_result = confmat(torch.tensor(final_predictions), torch.tensor(y_test_tensor)).numpy()\n",
        "print(\"Confusion Matrix:\\n\", confmat_result)\n",
        "\n",
        "# Plot the confusion matrix using seaborn heatmap\n",
        "plt.figure(figsize=(8, 6))\n",
        "sns.heatmap(confmat_result, annot=True, fmt='d', cmap='Blues', xticklabels=['No Crater', 'Crater'], yticklabels=['No Crater', 'Crater'])\n",
        "plt.title('Confusion Matrix')\n",
        "plt.ylabel('True Label')\n",
        "plt.xlabel('Predicted Label')\n",
        "plt.show()\n",
        "\n",
        "\n",
        "# ---- Compute F1-Score with torchmetrics ----\n",
        "f1_metric = torchmetrics.F1Score(task=\"multiclass\", num_classes=2, threshold=float(0.8500936271227173), average=\"macro\")\n",
        "f1_score_value = f1_metric(torch.tensor(final_predictions), torch.tensor(y_test_tensor)).item()\n",
        "\n",
        "# Print the torchmetrics F1 score\n",
        "print(f\"\\nMacro F1-Score using torchmetrics: {f1_score_value:.6f}\")\n",
        "\n",
        "# ---- Additional Metrics ----\n",
        "# Precision, Recall, F1-Score, Accuracy with new threshold\n",
        "report_dict = classification_report(y_test_tensor, final_predictions, output_dict=True, digits=6)\n",
        "#print(report_dict)\n",
        "print(f\"Macro Avg F1-Score: {report_dict['macro avg']['f1-score']:.6f}\\n\")\n",
        "print(classification_report(y_test_tensor, final_predictions, output_dict=False, digits=6))\n",
        "\n",
        "# AUC\n",
        "roc_auc_score_value = roc_auc_score(y_test_tensor, probabilities)\n",
        "print(f\"\\nAUC Score: {roc_auc_score_value:.6f}\")"
      ],
      "metadata": {
        "id": "LztZnj14gAXC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Self-labeling"
      ],
      "metadata": {
        "id": "fJRbxPu87eam"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the best model checkpoint based on the best trial number\n",
        "#best_trial_number = best_trial.number\n",
        "best_model_checkpoint_path = f\"./best_model_trial_2.ckpt\"\n",
        "print(f\"Loading the best model from: {best_model_checkpoint_path}\")\n",
        "\n",
        "# Load the model from the checkpoint\n",
        "task = ClassificationTask.load_from_checkpoint(\n",
        "    checkpoint_path=best_model_checkpoint_path,\n",
        "    criterion=nn.BCEWithLogitsLoss(pos_weight=class_weights_tensor[1]),\n",
        "    model=NewCNN()\n",
        ")\n",
        "task.eval()"
      ],
      "metadata": {
        "id": "JN1QAK4k7JcQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "confidence_threshold_pos = 0.99  # High confidence for class 1\n",
        "confidence_threshold_neg = 0.01  # High confidence for class 0\n",
        "\n",
        "pseudo_labels = []  # To store the pseudo labels (predicted labels)\n",
        "high_confidence_samples = []  # To store high-confidence data (samples)\n",
        "\n",
        "# Create a DataLoader for X_train_extra\n",
        "X_train_extra_loader = DataLoader(X_train_extra_reshaped, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "with torch.no_grad():\n",
        "    for batch in X_train_extra_loader:\n",
        "        batch = batch.to(task.device).float()  # Ensure the input is a float tensor\n",
        "        logits = task.model(batch / 255.0)  # Get model logits\n",
        "        probabilities = torch.sigmoid(logits).cpu().numpy()  # Apply sigmoid for probabilities\n",
        "\n",
        "        # Flatten probabilities to 1D array\n",
        "        probabilities = probabilities.flatten()\n",
        "\n",
        "        # Filter samples with high confidence in class 0 or class 1\n",
        "        high_conf_pos_mask = probabilities >= confidence_threshold_pos  # High-confidence for class 1\n",
        "        high_conf_neg_mask = probabilities <= confidence_threshold_neg  # High-confidence for class 0\n",
        "\n",
        "        # Combine the masks to get all high-confidence samples\n",
        "        high_confidence_mask = np.logical_or(high_conf_pos_mask, high_conf_neg_mask)\n",
        "\n",
        "        # Apply the mask to select only high-confidence samples\n",
        "        if np.any(high_confidence_mask):\n",
        "            high_conf_samples_probs = probabilities[high_confidence_mask]  # Get high-confidence probabilities\n",
        "            high_conf_samples_batch = batch[high_confidence_mask]  # Get high-confidence samples from the batch\n",
        "\n",
        "            # Store high-confidence samples\n",
        "            high_confidence_samples.append(high_conf_samples_batch.cpu().numpy())\n",
        "\n",
        "            # Assign pseudo-labels (0 or 1) based on confidence\n",
        "            confident_labels = (high_conf_samples_probs >= 0.5).astype(int)  # Label as 1 if above 0.5, otherwise 0\n",
        "            pseudo_labels.append(confident_labels)\n",
        "\n",
        "# Convert pseudo_labels and high-confidence samples to numpy arrays\n",
        "pseudo_labels = np.concatenate(pseudo_labels, axis=0)\n",
        "high_confidence_samples = np.concatenate(high_confidence_samples, axis=0)\n",
        "\n",
        "# Output some diagnostics\n",
        "print(\"Pseudo labels:\", pseudo_labels)\n",
        "print(f\"Original size: {X_train_extra_reshaped.shape}\")\n",
        "print(\"High confidence pseudo-labeled samples:\", high_confidence_samples.shape)\n",
        "print(\"Pseudo labels for high-confidence samples:\", pseudo_labels.shape)"
      ],
      "metadata": {
        "id": "Mmf6I75MBNcy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine X_train_reshaped with X_train_extra_reshaped\n",
        "X_combined = np.concatenate([X_train_reshaped, high_confidence_samples], axis=0)\n",
        "y_combined = np.concatenate([y_resampled, pseudo_labels], axis=0)\n",
        "\n",
        "print(f\"Combined training set shape: {X_combined.shape}, Combined labels shape: {y_combined.shape}\")"
      ],
      "metadata": {
        "id": "oJ108HnnBmt_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a new dataset with the combined data\n",
        "combined_dataset = CraterDataset(X_combined, y_combined, transform=train_transform)\n",
        "combined_loader = DataLoader(combined_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "# Calculate class weights to handle imbalance in the resampled training data\n",
        "class_weights = class_weight.compute_class_weight(\n",
        "    class_weight='balanced',\n",
        "    classes=np.unique(y_combined),\n",
        "    y=y_combined\n",
        ")\n",
        "class_weights_tensor = torch.tensor(class_weights, dtype=torch.float32)\n",
        "print(class_weights_tensor)\n",
        "\n",
        "# Set up the Optuna study and optimize the objective\n",
        "study = optuna.create_study(direction=\"maximize\")\n",
        "study.optimize(\n",
        "    lambda trial: Objective(combined_loader, val_loader, class_weights_tensor, trial),\n",
        "    n_trials=10 # Perform N trials of hyperparameter search\n",
        ")\n",
        "\n",
        "# Print the best hyperparameters found by Optuna\n",
        "best_trial = study.best_trial\n",
        "print(\"Best trial:\")\n",
        "print(f\"  F1 score: {best_trial.value}\")\n",
        "print(f\"  Params: \")\n",
        "for key, value in best_trial.params.items():\n",
        "    print(f\"    {key}: {value}\")"
      ],
      "metadata": {
        "id": "Sunj97vLBnNe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Final prediction"
      ],
      "metadata": {
        "id": "hl4r6MlYn570"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TestDataset(Dataset):\n",
        "    def __init__(self, images, transform=None):\n",
        "        self.images = images\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.images)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        image = self.images[idx]\n",
        "\n",
        "        # Convert numpy image to PIL image and apply transformations\n",
        "        image = Image.fromarray(image.squeeze(), mode='L')  # 'L' for grayscale\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        return image\n",
        "\n",
        "# Load the test data\n",
        "X_test = np.load(\"./data/Xtest1.npy\")\n",
        "X_test = X_test.reshape(-1, 1, 48, 48)  # Reshape to (num_samples, 1, 48, 48)\n",
        "\n",
        "# Define the transform\n",
        "test_transform = transforms.Compose([\n",
        "    transforms.ToTensor(),  # Convert the PIL image to a tensor\n",
        "])\n",
        "\n",
        "# Create the test dataset and loader\n",
        "test_dataset = TestDataset(X_test, transform=test_transform)\n",
        "test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)\n",
        "\n",
        "# Load the best model checkpoint\n",
        "best_model_checkpoint_path = f\"./best_model_trial_2.ckpt\"\n",
        "print(f\"Loading the best model from: {best_model_checkpoint_path}\")\n",
        "\n",
        "# Load the model from the checkpoint\n",
        "task = ClassificationTask.load_from_checkpoint(\n",
        "    checkpoint_path=best_model_checkpoint_path,\n",
        "    criterion=nn.BCEWithLogitsLoss(pos_weight=class_weights_tensor[1]),\n",
        "    model=NewCNN()\n",
        ")\n",
        "task.eval()\n",
        "\n",
        "# Make predictions on the test set\n",
        "logits_list = []\n",
        "with torch.no_grad():\n",
        "    for batch in test_loader:\n",
        "        batch = batch.to(task.device).float()\n",
        "        logits = task.model(batch)\n",
        "        logits_list.append(logits.cpu().numpy())\n",
        "\n",
        "logits = np.concatenate(logits_list, axis=0)\n",
        "probabilities = torch.sigmoid(torch.tensor(logits)).numpy()\n",
        "predictions = (probabilities >= 0.8500936271227173).astype(int).flatten()\n",
        "\n",
        "print(predictions)\n",
        "np.save(\"y_pred\", predictions)"
      ],
      "metadata": {
        "id": "Wu_KARdSn5Uc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "!pip install PyPDF2\n",
        "from PyPDF2 import PdfMerger\n",
        "from PIL import Image\n",
        "import os\n",
        "\n",
        "# Load the test data (X_test.npy) and predictions\n",
        "X_test = np.load(\"./data/Xtest1.npy\")\n",
        "X_test = X_test.reshape(-1, 1, 48, 48)  # Reshape to (num_samples, 1, 48, 48)\n",
        "\n",
        "predictions = y_pred\n",
        "\n",
        "num_images = X_test.shape[0]\n",
        "img_shape = (48, 48)\n",
        "cols = 10\n",
        "images_per_batch = 100\n",
        "rows = int(np.ceil(images_per_batch / cols))\n",
        "batch_image_paths = []\n",
        "\n",
        "for batch_start in range(0, num_images, images_per_batch):\n",
        "    fig, axes = plt.subplots(rows, cols, figsize=(cols * 2, rows * 2))\n",
        "    axes = axes.flatten()\n",
        "\n",
        "    for i, ax in enumerate(axes):\n",
        "        img_idx = batch_start + i\n",
        "        if img_idx < num_images:\n",
        "            img = X_test[img_idx, :].reshape(img_shape)\n",
        "            ax.imshow(img, cmap='gray')\n",
        "            ax.set_title(f'ID: {img_idx}, Pred: {predictions[img_idx]}')\n",
        "            ax.axis('off')\n",
        "        else:\n",
        "            ax.axis('off')\n",
        "\n",
        "    batch_filename = f'batch_{batch_start // images_per_batch}.png'\n",
        "    plt.savefig(batch_filename, bbox_inches='tight', pad_inches=0.1, dpi=120)\n",
        "    batch_image_paths.append(batch_filename)\n",
        "    plt.close(fig)\n",
        "\n",
        "# Convert images to PDFs and merge them\n",
        "batch_pdfs = []\n",
        "for batch_image in batch_image_paths:\n",
        "    img = Image.open(batch_image)\n",
        "    if img.mode != 'RGB':\n",
        "        img = img.convert('RGB')\n",
        "    img_resized = img.resize((img.width // 2, img.height // 2), Image.Resampling.LANCZOS)\n",
        "    pdf_filename = batch_image.replace('.png', '.pdf')\n",
        "    img_resized.save(pdf_filename, 'PDF', resolution=100.0, quality=100, optimize=True)\n",
        "    batch_pdfs.append(pdf_filename)\n",
        "\n",
        "# Merge all PDFs into a single document\n",
        "pdf_merger = PdfMerger()\n",
        "for pdf_file in batch_pdfs:\n",
        "    pdf_merger.append(pdf_file)\n",
        "\n",
        "final_pdf_filename = 'X_test_images_with_predictions.pdf'\n",
        "pdf_merger.write(final_pdf_filename)\n",
        "pdf_merger.close()\n",
        "\n",
        "print(f\"Final merged PDF saved as '{final_pdf_filename}'\")\n",
        "\n",
        "# Clean up: remove batch images and PDFs\n",
        "for batch_image in batch_image_paths:\n",
        "    os.remove(batch_image)\n",
        "for pdf_file in batch_pdfs:\n",
        "    os.remove(pdf_file)\n",
        "\n",
        "print(\"Intermediate batch images and PDF files deleted.\")"
      ],
      "metadata": {
        "id": "7EUfHQsHqaAa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Plots"
      ],
      "metadata": {
        "id": "Y1CTrv8zf_4k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf ~/.tensorboard-info\n",
        "%reload_ext tensorboard\n",
        "%tensorboard --logdir=lightning_logs/"
      ],
      "metadata": {
        "id": "6uAY8n_WtNP1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorboard.backend.event_processing import event_accumulator\n",
        "\n",
        "# Specify the path to the TensorBoard log file\n",
        "log_dir = f\"lightning_logs/optuna_trials/trial_{best_trial_number}/version_0\"\n",
        "\n",
        "# Load TensorBoard event data\n",
        "event_acc = event_accumulator.EventAccumulator(log_dir)\n",
        "event_acc.Reload()\n",
        "\n",
        "# Extract scalars for train and validation metrics\n",
        "train_loss = event_acc.Scalars('train_loss_epoch')\n",
        "train_acc = event_acc.Scalars('train_acc_epoch')\n",
        "val_loss = event_acc.Scalars('val_loss_epoch')\n",
        "val_acc = event_acc.Scalars('val_acc_epoch')\n",
        "\n",
        "# Extract epoch numbers and metric values (filtering by epoch end logs)\n",
        "epochs = [i for i in range(len(train_loss))]\n",
        "\n",
        "# Prepare metric values for train\n",
        "train_loss_values = [x.value for x in train_loss]\n",
        "train_acc_values = [x.value for x in train_acc]\n",
        "\n",
        "# Prepare metric values for validation\n",
        "val_loss_values = [x.value for x in val_loss]\n",
        "val_acc_values = [x.value for x in val_acc]\n",
        "\n",
        "# Create plot for training metrics\n",
        "fig, ax1 = plt.subplots(figsize=(8, 6))\n",
        "\n",
        "# Plot Training Loss on the left y-axis\n",
        "ax1.set_xlabel('Epochs')\n",
        "ax1.set_ylabel('Training Loss', color='tab:red')\n",
        "ax1.plot(epochs, train_loss_values, color='tab:red', label='Training Loss')\n",
        "ax1.tick_params(axis='y', labelcolor='tab:red')\n",
        "\n",
        "# Create a second y-axis for training accuracy\n",
        "ax2 = ax1.twinx()\n",
        "ax2.set_ylabel('Training Accuracy', color='tab:blue')\n",
        "ax2.plot(epochs, train_acc_values, color='tab:blue', label='Training Accuracy')\n",
        "ax2.tick_params(axis='y', labelcolor='tab:blue')\n",
        "\n",
        "# Add a title to the plot for training metrics\n",
        "plt.title('Training Loss and Accuracy per Epoch')\n",
        "\n",
        "# Show the plot for training\n",
        "fig.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "# Create plot for validation metrics\n",
        "fig, ax1 = plt.subplots(figsize=(8, 6))\n",
        "\n",
        "# Plot Validation Loss on the left y-axis\n",
        "ax1.set_xlabel('Epochs')\n",
        "ax1.set_ylabel('Validation Loss', color='tab:red')\n",
        "ax1.plot(epochs, val_loss_values, color='tab:red', label='Validation Loss')\n",
        "ax1.tick_params(axis='y', labelcolor='tab:red')\n",
        "\n",
        "# Create a second y-axis for validation accuracy\n",
        "ax2 = ax1.twinx()\n",
        "ax2.set_ylabel('Validation Accuracy', color='tab:blue')\n",
        "ax2.plot(epochs, val_acc_values, color='tab:blue', label='Validation Accuracy')\n",
        "ax2.tick_params(axis='y', labelcolor='tab:blue')\n",
        "\n",
        "# Add a title to the plot for validation metrics\n",
        "plt.title('Validation Loss and Accuracy per Epoch')\n",
        "\n",
        "# Show the plot for validation\n",
        "fig.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "bWDbE2eXbvHJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Clean-up"
      ],
      "metadata": {
        "id": "EuJbhhMV7BbT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "def delete_files_in_directory(directory):\n",
        "    for filename in os.listdir(directory):\n",
        "        file_path = os.path.join(directory, filename)\n",
        "        try:\n",
        "            if os.path.isfile(file_path) or os.path.islink(file_path):\n",
        "                os.remove(file_path)\n",
        "            elif os.path.isdir(file_path):\n",
        "                for root, dirs, files in os.walk(file_path, topdown=False):\n",
        "                    for name in files:\n",
        "                        os.remove(os.path.join(root, name))\n",
        "                    for name in dirs:\n",
        "                        os.rmdir(os.path.join(root, name))\n",
        "                os.rmdir(file_path)\n",
        "        except Exception as e:\n",
        "            print(f\"Failed to delete {file_path}. Reason: {e}\")\n",
        "\n",
        "directory = 'lightning_logs/optuna_trials'\n",
        "delete_files_in_directory(directory)"
      ],
      "metadata": {
        "id": "ypRMmRDK50lG"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## SVC Model Training"
      ],
      "metadata": {
        "id": "F6_AoOUJ8X_c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit\n",
        "\n",
        "from sklearn import datasets\n",
        "from matplotlib import cm\n",
        "\n",
        "from sklearn.metrics import ConfusionMatrixDisplay\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import (\n",
        "    roc_curve, auc, precision_recall_curve, roc_auc_score,\n",
        "    precision_score, recall_score, f1_score, classification_report\n",
        ")\n",
        "\n",
        "# Load data\n",
        "X_train = np.load(\"./data/Xtrain1.npy\")\n",
        "y_train = np.load(\"./data/Ytrain1.npy\")\n",
        "X_train_extra = np.load(\"./data/Xtrain1_extra.npy\")\n",
        "\n",
        "print(f\"X_train shape: {X_train.shape}\")\n",
        "print(f\"y_train shape: {y_train.shape}\")\n",
        "print(f\"X_train_extra shape: {X_train_extra.shape}\")\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=42)\n",
        "\n",
        "# Define the parameter grid\n",
        "C_range = [0.1, 1]\n",
        "gamma_range = [0.1, 1]\n",
        "param_grid = dict(C=C_range, gamma=gamma_range, kernel=['rbf'])\n",
        "\n",
        "# Use StratifiedShuffleSplit for better handling of unbalanced data\n",
        "cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)\n",
        "\n",
        "# Initialize the SVM classifier with class balancing\n",
        "model = SVC(class_weight='balanced')\n",
        "\n",
        "# Setup GridSearchCV with F1 scoring and refit based on F1 score\n",
        "grid_search = GridSearchCV(\n",
        "    estimator=model,\n",
        "    param_grid=param_grid,\n",
        "    scoring='f1',  # Use 'f1' for binary classification\n",
        "    cv=5,          # Number of folds for cross-validation\n",
        "    refit='f1',    # Refit on the model that maximizes the F1 score\n",
        "    verbose=0\n",
        ")\n",
        "\n",
        "scaler = StandardScaler()\n",
        "X_train_scaled = scaler.fit_transform(X_train)\n",
        "X_test_scaled = scaler.transform(X_test)\n",
        "\n",
        "# Fit the grid search\n",
        "grid_search.fit(X_train_scaled, y_train)\n",
        "\n",
        "# Best parameters and score\n",
        "print(\"Best Parameters:\", grid_search.best_params_)\n",
        "print(\"Best F1 Score:\", grid_search.best_score_)\n",
        "\n",
        "# Predict on the test set using the best model\n",
        "y_pred = grid_search.best_estimator_.predict(X_test)\n",
        "\n",
        "report_dict = classification_report(y_test, y_pred, output_dict=True, digits=6, zero_division=1)\n",
        "print(f\"Weighted Avg F1-Score: {report_dict['weighted avg']['f1-score']}\")\n",
        "\n",
        "ConfusionMatrixDisplay.from_estimator(grid_search.best_estimator_, X_test, y_test)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "YnTtNrvC8b7W"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "w9nV0Gge8nd_",
        "1jMSJm45b-eZ",
        "Y1CTrv8zf_4k",
        "F6_AoOUJ8X_c"
      ],
      "gpuType": "T4",
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}