Rakuten_modele(1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Rakuten_modele.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KFpSDQdgAzOM",
        "outputId": "8c9af649-5a6a-47ce-da37-98e2491359b8"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install torch torchvision\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (4.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.7)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tokenizers==0.9.4 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.7.0+cu101)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.8.1+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.18.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch) (0.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch) (0.16.0)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (7.0.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (0.1.94)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WEJnN-1PBrV4",
        "outputId": "c43a196c-ae62-4392-bdf0-0c595054f70d"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np \n",
        "import nltk\n",
        "from nltk import WordPunctTokenizer\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "import seaborn as sns\n",
        "import os\n",
        "import torch\n",
        "from transformers import CamembertModel, CamembertTokenizer, CamembertConfig\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "from sklearn.pipeline import FeatureUnion, Pipeline, make_pipeline\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "import torch.nn as nn \n",
        "from skimage import io, transform\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import models, transforms, utils\n",
        "import random\n",
        "import cv2\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.utils.class_weight import compute_class_weight"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgomrPKmBp8m",
        "outputId": "9e71b76d-eb32-427f-fe95-e1da2a887933"
      },
      "source": [
        "from google.colab import drive,files\n",
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "egoFX4t9D2vP",
        "outputId": "49b1dcec-9041-403f-c811-9e2dd5d9d5c5"
      },
      "source": [
        "!unzip /content/drive/MyDrive/DS2021/Datasets/archive\\(8\\).zip "
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/MyDrive/DS2021/Datasets/archive(8).zip\n",
            "replace X_test_update.csv? [y]es, [n]o, [A]ll, [N]one, [r]ename: N\n",
            "N\n",
            "N\n",
            "N\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQ4BEz1ED3wd"
      },
      "source": [
        "x_train = pd.read_csv(\"/content/X_train_update.csv\")\n",
        "y_train = pd.read_csv(\"/content/Y_train_CVw08PX.csv\")\n",
        "x_test = pd.read_csv(\"/content/X_test_update.csv\")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPhsmO0ZEH1S"
      },
      "source": [
        "x_train.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)\n",
        "y_train.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)\n",
        "x_test.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N0qhpi6-lLQO"
      },
      "source": [
        "Helpers function, Classes\n",
        "\n",
        "```\n",
        "# This is formatted as code\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91VfE49sFs0H"
      },
      "source": [
        "class simple_Text_cleaner(BaseEstimator, TransformerMixin):\n",
        "  # add another additional parameter, just for fun, while we are at it\n",
        "    def __init__(self, stopwords,columns,tokenizer=WordPunctTokenizer()): \n",
        "        self.columns = columns\n",
        "        self.stopwords = stopwords\n",
        "        self.tokenizer=tokenizer\n",
        "    def rm_stopwords(self,tokens):\n",
        "        tokens=self.tokenizer.tokenize(tokens)\n",
        "        return [ tk for tk in tokens if  tk not in self.stopwords ]\n",
        "    def text_clean_up(self,s=\"\"):\n",
        "        import re\n",
        "        user_pattern       = '@[^\\s]+'\n",
        "        s=re.sub(user_pattern, \"\", s)\n",
        "        remove = '\"#$%&()*+/:;<=>@[\\\\]^_`{|}~”“'\n",
        "        pattern = r\"[{}]\".format(remove)\n",
        "        s=re.sub(pattern,' ', s) \n",
        "        sequencePattern   = r\"(.)\\1\\1+\"\n",
        "        seqReplacePattern = r\"\\1\\1\"\n",
        "        s = re.sub(sequencePattern, seqReplacePattern, s)\n",
        "        s = re.sub(\"<[^>]*>\",' ', s)\n",
        "        s = re.sub(\"[\\r\\n]+\",' ', s)\n",
        "        s = re.sub(\"http\\S+\",' ', s)\n",
        "        s = re.sub(\"\\$[^>]*\\$\",' ', s)\n",
        "        s = re.sub(\"\\d+\",' ', s)\n",
        "        s = re.sub(\"\\s\\s+\",' ', s)\n",
        "        s.strip()\n",
        "        return s\n",
        "    def fit(self, X, y=None):\n",
        "        return self\n",
        "    def transform(self, X, y=None):\n",
        "        X_ = X.copy()\n",
        "        X_[\"image_path\"] = (X_[\"imageid\"].apply(lambda x: str(int(x)))+\"_product_\"+X_[\"productid\"].apply(lambda x: str(int(x)))+\".jpg\").apply(lambda x: \"image_\"+x)\n",
        "        for i in self.columns:\n",
        "            X_[i] = X_[i].apply(lambda x: \" \".join(self.rm_stopwords(self.text_clean_up(x))))\n",
        "        return X_"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gdT0SN64GIxp"
      },
      "source": [
        "def encode_reviews(tokenizer, model, device, cpu, reviews, max_length):\n",
        "    token_ids = torch.tensor([])\n",
        "    input_ids=torch.tensor([],dtype=torch.long)\n",
        "    attention_mask=torch.tensor([],dtype=torch.long)\n",
        "    h = 0\n",
        "    for i, review in enumerate(reviews):\n",
        "        encoded = encoded_text = tokenizer.encode_plus(\n",
        "                        review,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = max_length,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "        input_ids=torch.cat((input_ids,encoded_text['input_ids']),dim=0)\n",
        "        attention_mask=torch.cat((attention_mask,encoded_text['attention_mask']),dim=0)\n",
        "        h+=1\n",
        "        if h == 128:\n",
        "            hidden = model(input_ids.to(device),attention_mask=attention_mask.to(device))[2][-2]\n",
        "            token_ids=torch.cat((token_ids,torch.mean(hidden, dim=1).to(cpu)),dim=0)\n",
        "            h=0\n",
        "            input_ids=torch.tensor([],dtype=torch.long)\n",
        "            attention_mask=torch.tensor([],dtype=torch.long)\n",
        "    if input_ids != torch.tensor([],dtype=torch.long):\n",
        "        hidden = model(input_ids.to(device),attention_mask=attention_mask.to(device))[2][-2]\n",
        "        token_ids=torch.cat((token_ids,torch.mean(hidden, dim=1).to(cpu)),dim=0)\n",
        "        print(token_ids.size())\n",
        "    token_ids=token_ids.numpy()\n",
        "    print(token_ids.shape)\n",
        "    return token_ids\n",
        "\n",
        "class CamembertPreprocessor(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self, max_seq_length,column):\n",
        "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "        self.cpu=torch.device(\"cpu\")\n",
        "        print(self.device)\n",
        "        self.tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert =CamembertModel.from_pretrained(\"camembert-base\", config=self.config).to(self.device)\n",
        "        for p in self.camembert.parameters():\n",
        "                p.requires_grad_(False)\n",
        "        self.max_seq_length = max_seq_length\n",
        "        self.column = column\n",
        "        self.camembert.eval()\n",
        "    def fit(self, X=None):\n",
        "        return self \n",
        "    \n",
        "    def transform(self, X, y=None):\n",
        "        # 1. Tokenize\n",
        "        X_encoded = encode_reviews(self.tokenizer,self.camembert,self.device,self.cpu, X[self.column].values, self.max_seq_length)\n",
        "        return X_encoded     \n",
        "    \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b4DmnNcp6KFt",
        "outputId": "31fd5910-462a-443e-e49f-e0ccf8e7b0fe"
      },
      "source": [
        "\n",
        "torch.cat((torch.tensor([]),torch.tensor([1,2])),dim=0)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1., 2.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1WTwE-WGI7r"
      },
      "source": [
        "class Second_to_last_SentenceEmbedding(nn.Module):\n",
        "    def __init__(self):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert = CamembertModel.from_pretrained(\"camembert-base\", config=self.config)\n",
        "        self.dense_layer = nn.Sequential(nn.Linear(768,100),nn.ReLU(),nn.Dropout(p=0.2))\n",
        "\n",
        "        self.fc2=nn.Linear(100,27)\n",
        "        for p in self.camembert.parameters():\n",
        "            p.requires_grad_(False)\n",
        "    def forward(self, input,attention_mask=None):\n",
        "        hidden = self.camembert(input,attention_mask=attention_mask)[2]\n",
        "        token_vecs = hidden[-2]\n",
        "        x = torch.mean(token_vecs,dim=1)\n",
        "        h = self.dense_layer(x)\n",
        "        return self.fc2(h)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcZS8g4NGJEJ"
      },
      "source": [
        "class Columns_Selector(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self,column):\n",
        "        self.column = column\n",
        "    def fit(self, X = None):\n",
        "        return self \n",
        "    def transform(self, X, y=None):\n",
        "        return X[self.column]     \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pal4xfSiGJLT"
      },
      "source": [
        "class To_dense(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self):\n",
        "        pass\n",
        "    def fit(self, X = None):\n",
        "        return self \n",
        "    def transform(self, X, y=None):\n",
        "        X = X.toarray()\n",
        "        print(X.shape)\n",
        "        return X \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oS7yq_DWHRIQ"
      },
      "source": [
        "class MultimodalTruncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
        "Dataset(Dataset):\n",
        "    \"\"\"Face Landmarks dataset.\"\"\"\n",
        "\n",
        "    def __init__(self,tokenizer, input_csv_file, root_dir,max_length=60 , transform=None,Y_csv_file=None):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            csv_file (string): Path to the csv file with annotations.\n",
        "            root_dir (string): Directory with all the images.\n",
        "            transform (callable, optional): Optional transform to be applied\n",
        "                on a sample.\n",
        "        \"\"\"\n",
        "        if type(input_csv_file) == str:\n",
        "            self.input_file = pd.read_csv(input_csv_file)\n",
        "        else:\n",
        "            self.input_file = input_csv_file\n",
        "        self.input_file.rename(columns={\"Unnamed: 0\": \"Id\"}, inplace=True)\n",
        "        self.root_dir = root_dir\n",
        "        self.transform = transform\n",
        "        self.max_length=max_length\n",
        "        self.input_file[\"image_path\"]=(self.input_file[\"imageid\"].apply(lambda x: str(int(x)))+\"_product_\"+self.input_file[\"productid\"].apply(lambda x: str(int(x)))+\".jpg\").apply(lambda x: \"image_\"+x)\n",
        "        if Y_csv_file !=None:\n",
        "            if type(Y_csv_file) == str:\n",
        "                self.output = pd.read_csv(Y_csv_file)\n",
        "                \n",
        "            else:\n",
        "                self.output = Y_csv_file\n",
        "            self.output.rename(columns={\"Unnamed: 0\": \"Id\"}, inplace=True)\n",
        "            self.classes = list(set(self.output[\"prdtypecode\"].values))\n",
        "        else:\n",
        "            self.ouptut = None\n",
        "    def __len__(self):\n",
        "        return len(self.input_file)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        if torch.is_tensor(idx):\n",
        "            idx = idx.tolist()\n",
        "        \n",
        "        img_name = os.path.join(self.root_dir,\n",
        "                                self.input_file[\"image_path\"].iloc[idx])\n",
        "        image = io.imread(img_name)\n",
        "        text = self.input_file[\"designation\"].iloc[idx]\n",
        "        encoded_text = tokenizer.encode_plus(\n",
        "                        text,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "        sample = {'image': image, 'input_ids':encoded_text['input_ids'] ,'attention_mask':encoded_text['attention_mask']}\n",
        "\n",
        "        if self.transform:\n",
        "            sample['image'] = self.transform(sample['image'])\n",
        "        if self.output is None:\n",
        "            return sample\n",
        "        sample[\"label\"]=self.classes.index(self.output[\"prdtypecode\"].loc[self.input_file[\"Id\"].iloc[idx]])\n",
        "        return sample"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OgApZ_05HZrV"
      },
      "source": [
        "class Microscope:\n",
        "    \"\"\"\n",
        "    Cutting out the edges around the center circle of the image\n",
        "    Imitating a picture, taken through the microscope\n",
        "\n",
        "    Args:\n",
        "        p (float): probability of applying an augmentation\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, p: float = 0.5):\n",
        "        self.p = p\n",
        "\n",
        "    def __call__(self, img):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            img (PIL Image): Image to apply transformation to.\n",
        "\n",
        "        Returns:NumPyNumPy\n",
        "            PIL Image: Image with transformation.\n",
        "        \"\"\"\n",
        "        if random.random() < self.p:\n",
        "            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n",
        "                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n",
        "                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n",
        "                        (0, 0, 0), # color\n",
        "                        -1)\n",
        "\n",
        "            mask = circle - 255\n",
        "            img = np.multiply(img, mask)\n",
        "        \n",
        "        return img\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(p={self.p})'"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WLZuV956HxZj"
      },
      "source": [
        "class Feature_Extraction(nn.Module):\n",
        "    # deuxieme dimension de l'output de chaque tenseurs a la sortie d'un layer de resnet\n",
        "    resnet_caracteristics=[64,64,64,64,256,512,1024,2048,2048]\n",
        "    pooling_target=[(4,8),(4,8),(4,8),(4,8),(2,4),(2,2),(1,2),(1,1),(1,1)]\n",
        "    embedding_strategy={\"Second_to_last_average\":768,\"Start_token_embedding\":768,\"last_four_embedding_average\":3072}\n",
        "    def __init__(self,resnet_layers=-1,to_tune=False,sentence_embedding=\"Second_to_last_average\"):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert =CamembertModel.from_pretrained(\"camembert-base\", config=self.config)\n",
        "        self.Resnet = models.resnet50(pretrained=True)\n",
        "        self.Resnet = nn.Sequential(*list(self.Resnet.children())[:resnet_layers])\n",
        "        self.pooling = nn.AdaptiveMaxPool2d(self.pooling_target[resnet_layers-1])\n",
        "        self.flat = nn.Flatten()\n",
        "        self.strategy = sentence_embedding\n",
        "        if sentence_embedding not in self.embedding_strategy:\n",
        "            self.strategy = \"Second_to_last_average\"    \n",
        "        if self.strategy == \"Start_token_embedding\":\n",
        "            to_tune = True\n",
        "        if to_tune == False:\n",
        "            for p in self.camembert.parameters():\n",
        "                p.requires_grad_(False)\n",
        "            for p in self.Resnet.parameters():\n",
        "                p.requires_grad_(False)\n",
        "        self.output_size = 2048+ self.embedding_strategy[self.strategy]\n",
        "    def sentence_embedding(self,hiddens):\n",
        "        # we get the output of the second to last hidden layers and average it over all token\n",
        "        if self.strategy == \"Second_to_last_average\":\n",
        "            return torch.mean(hiddens[-2],dim=1) \n",
        "        # we use the first token embedding  from the ouptut last hidden layers \n",
        "        # fine_tune should be true \n",
        "        elif self.strategy == \"Start_token_embedding\":\n",
        "            return hiddens[-1].permute(1,0,2)[0]\n",
        "        # we use the last four hidden layer average and concatenate them\n",
        "        elif self.strategy == \"last_four_embedding_average\":\n",
        "            x=torch.cat((hiddens[-4], hiddens[-3], hiddens[-2], hiddens[-1]), dim = 2)\n",
        "            return torch.mean(x, dim=1)\n",
        "    def forward(self, input, image, attention_mask=None):\n",
        "        hiddens = self.camembert(input,attention_mask = attention_mask)[2]\n",
        "        embeddings = self.sentence_embedding(hiddens)\n",
        "        x = self.Resnet(image)\n",
        "        h = self.pooling(x)\n",
        "        return torch.cat((embeddings, self.flat(h)),dim=1)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cgcK01jlH6bC"
      },
      "source": [
        "class Multimodal_Dense_model(nn.Module):\n",
        "    def __init__(self,dropout=0.2,resnet_layers=-1,to_tune=False,sentence_embedding=\"Second_to_last_average\"):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.feature_extractor = Feature_Extraction(resnet_layers=resnet_layers,to_tune=to_tune,sentence_embedding=sentence_embedding)\n",
        "        self.input_size = self.feature_extractor.output_size\n",
        "        self.dense_layer = nn.Sequential(OrderedDict([\n",
        "          ('dense1', nn.Linear(in_features=self.input_size,out_features=768)),\n",
        "          ('relu1', nn.ReLU()),\n",
        "          ('dropout1', nn.Dropout(p=0.2)),\n",
        "          ('dense2', nn.Linear(in_features=768,out_features=256)),\n",
        "          ('relu2', nn.ReLU()),\n",
        "          ('dropout2', nn.Dropout(p=0.2)),\n",
        "          ('dense3', nn.Linear(in_features=256,out_features=64)),\n",
        "          ('relu3', nn.ReLU()),\n",
        "          ('dropout3', nn.Dropout(p=0.2))]))\n",
        "        self.fc = nn.Linear(64,27)\n",
        "    def forward(self, input,image,attention_mask=None):\n",
        "        feature = self.feature_extractor(input=input,image=image,attention_mask=attention_mask)\n",
        "        h = self.dense_layer(feature)\n",
        "        return self.fc(h)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sUaOu1RCIYk6"
      },
      "source": [
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        Microscope(p=0.5),\n",
        "        transforms.ToPILImage(),\n",
        "        transforms.RandomResizedCrop(size=384, scale=(0.8, 1.0)),\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        transforms.RandomVerticalFlip(),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'val': transforms.Compose([\n",
        "        transforms.Resize(256),\n",
        "        transforms.CenterCrop(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        transforms.ToPILImage(),\n",
        "        transforms.Resize(512),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "}"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJ9fwFN5IBfC"
      },
      "source": [
        "tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
        "training_set = MultimodalDataset(tokenizer=tokenizer, input_csv_file=\"/content/X_train_update.csv\", root_dir=\"/content/images/images/image_train\",max_length=60 , transform=data_transforms[\"train\"],Y_csv_file=\"/content/Y_train_CVw08PX.csv\")\n",
        "def make_weights_for_balanced_classes(images, nclasses):                                                                            \n",
        "    count = list(images.output[\"prdtypecode\"].value_counts().loc[images.classes])                                    \n",
        "    N = float(sum(count)) \n",
        "    weight_per_class=[0]*nclasses                                                  \n",
        "    for i in range(nclasses):                                                   \n",
        "        weight_per_class[i] = N/float(count[i])                                 \n",
        "    weight = [0] * len(images)                                              \n",
        "    for idx in range(int(N)):    \n",
        "        h=images.classes.index(images.output[\"prdtypecode\"].iloc[idx])                                      \n",
        "        weight[idx] = weight_per_class[h]                                  \n",
        "    return weight\n",
        "weights = make_weights_for_balanced_classes(training_set, len(training_set.classes))                                                                \n",
        "weights = torch.DoubleTensor(weights)                                       \n",
        "sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights))                     \n",
        "                                                                                                                                                                        \n",
        "train_dl = torch.utils.data.DataLoader(training_set, batch_size=16,                              \n",
        "                                                             sampler = sampler, num_workers=4)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vIv1iRG8Ku6l"
      },
      "source": [
        "Baseline model\n",
        "Tf-idfVectorizer + Logistics Regression (Texte)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1dA388wCFyG3"
      },
      "source": [
        "X_train,X_Val,Y_train,Y_Val=train_test_split(x_train, y_train, test_size = 0.2, shuffle = True , random_state = 155)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZ8cYRrEZCSf"
      },
      "source": [
        "target = Y_train[\"prdtypecode\"]"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UWQ66Li2iAFE"
      },
      "source": [
        "weights=compute_class_weight(\"balanced\", classes= np.unique(target), y=target)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "beYpPUcBjeqN",
        "outputId": "4b7c6c01-0112-42e4-f44f-cd85cb0d53ff"
      },
      "source": [
        "len(h)"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "27"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fl9jrCVUi9Hr"
      },
      "source": [
        "h=np.unique(target)\n",
        "weights_dict={}\n",
        "for i in range(len(h)):\n",
        "    weights_dict[h[i]] = weights[i]\n",
        "    "
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eNGe90VVlsqd"
      },
      "source": [
        "?LogisticRegression"
      ],
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fIkxcZhC8IjR"
      },
      "source": [
        "stop_words = stopwords.words('french')\n",
        "stop_words.extend(stopwords.words('english'))"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NvZL6ieUGH0i"
      },
      "source": [
        "Base_model = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('column_selector', Columns_Selector(column = \"designation\")),\n",
        "                               ('Vectorization', TfidfVectorizer(stop_words = stop_words, max_features = 5000)),\n",
        "                               ('to_dense', To_dense()),\n",
        "                               ('scaler', StandardScaler()),\n",
        "                               ('model', LogisticRegression(multi_class='multinomial',class_weight= \"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3fZ7aiHPUib2",
        "outputId": "d54786c3-7708-4962-f2f9-127c01c21dc0"
      },
      "source": [
        "Base_model.fit(X_train,target)"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(67932, 5000)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('cleaner',\n",
              "                 simple_Text_cleaner(columns=['designation'],\n",
              "                                     stopwords=['au', 'aux', 'avec', 'ce',\n",
              "                                                'ces', 'dans', 'de', 'des',\n",
              "                                                'du', 'elle', 'en', 'et', 'eux',\n",
              "                                                'il', 'ils', 'je', 'la', 'le',\n",
              "                                                'les', 'leur', 'lui', 'ma',\n",
              "                                                'mais', 'me', 'même', 'mes',\n",
              "                                                'moi', 'mon', 'ne', 'nos', ...],\n",
              "                                     tokenizer=WordPunctTokenizer(pattern='\\\\w+|[^\\\\w\\\\s]+', gaps=False, discard_empt...\n",
              "                ('to_dense', To_dense()),\n",
              "                ('scaler',\n",
              "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
              "                ('model',\n",
              "                 LogisticRegression(C=1.0, class_weight='balanced', dual=False,\n",
              "                                    fit_intercept=True, intercept_scaling=1,\n",
              "                                    l1_ratio=None, max_iter=100,\n",
              "                                    multi_class='multinomial', n_jobs=None,\n",
              "                                    penalty='l2', random_state=None,\n",
              "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                                    warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5b5yoJsiWm4r",
        "outputId": "231fad1f-71b3-4441-fd09-2c43372466a6"
      },
      "source": [
        "pred=Base_model.predict(X_Val)"
      ],
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(16984, 5000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jLUb-R9UweH",
        "outputId": "9ae67224-59b9-48c4-83ee-ea3516a495a8"
      },
      "source": [
        "print(classification_report(pred,Y_Val[\"prdtypecode\"]))"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.32      0.34      0.33       545\n",
            "          40       0.59      0.59      0.59       517\n",
            "          50       0.69      0.69      0.69       338\n",
            "          60       0.83      0.73      0.77       175\n",
            "        1140       0.60      0.60      0.60       515\n",
            "        1160       0.82      0.90      0.86       719\n",
            "        1180       0.53      0.40      0.46       197\n",
            "        1280       0.53      0.59      0.56       928\n",
            "        1281       0.45      0.34      0.39       497\n",
            "        1300       0.83      0.87      0.85       989\n",
            "        1301       0.89      0.87      0.88       145\n",
            "        1302       0.71      0.73      0.72       480\n",
            "        1320       0.63      0.60      0.61       662\n",
            "        1560       0.77      0.78      0.78      1071\n",
            "        1920       0.87      0.85      0.86       893\n",
            "        1940       0.72      0.72      0.72       189\n",
            "        2060       0.69      0.73      0.71       923\n",
            "        2220       0.64      0.69      0.66       140\n",
            "        2280       0.59      0.62      0.61       882\n",
            "        2403       0.62      0.69      0.66       857\n",
            "        2462       0.65      0.66      0.66       306\n",
            "        2522       0.84      0.89      0.86       931\n",
            "        2582       0.62      0.59      0.60       535\n",
            "        2583       0.94      0.94      0.94      2047\n",
            "        2585       0.67      0.64      0.66       499\n",
            "        2705       0.51      0.34      0.41       844\n",
            "        2905       0.98      0.99      0.99       160\n",
            "\n",
            "    accuracy                           0.71     16984\n",
            "   macro avg       0.69      0.68      0.68     16984\n",
            "weighted avg       0.70      0.71      0.70     16984\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c9eobnxLcd6r"
      },
      "source": [
        "Bert extraction De feature "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FiWBXb1UcacO",
        "outputId": "739be87c-cc63-4708-a978-1bda5668490e"
      },
      "source": [
        "Simple_text_model = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('Vectorization', CamembertPreprocessor(max_seq_length = 64, column = \"designation\")),\n",
        "                               ('model', LogisticRegression(multi_class='multinomial',class_weight= \"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5cebwAKptA2R",
        "outputId": "1412da3b-b502-4180-dcd6-eee3bb64ab54"
      },
      "source": [
        "Simple_text_model.fit(X_train,target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}