notebooks/3.2 Bert (Name + Desc) Full Training.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-08-25T05:47:26.912614Z","iopub.status.busy":"2024-08-25T05:47:26.912296Z","iopub.status.idle":"2024-08-25T05:47:33.145411Z","shell.execute_reply":"2024-08-25T05:47:33.144610Z","shell.execute_reply.started":"2024-08-25T05:47:26.912579Z"},"trusted":true},"outputs":[],"source":["import pickle\n","import numpy as np\n","import pandas as pd\n","from sklearn.metrics import precision_recall_curve, auc, roc_auc_score\n","from sklearn.model_selection import KFold\n","from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_linear_schedule_with_warmup\n","import torch\n","import gc\n","import re\n","from tqdm import tqdm\n","\n","tqdm.pandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-25T05:47:33.148664Z","iopub.status.busy":"2024-08-25T05:47:33.147504Z","iopub.status.idle":"2024-08-25T05:47:41.810967Z","shell.execute_reply":"2024-08-25T05:47:41.809966Z","shell.execute_reply.started":"2024-08-25T05:47:33.148618Z"},"trusted":true},"outputs":[],"source":["text_and_bert = pd.read_parquet('/kaggle/extracted_data/text_and_bert.parquet', engine='pyarrow')\n","text_and_bert['description'] = text_and_bert['description'].fillna('no desc')"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["attrs = pd.read_parquet('/kaggle/input/extracted_data/attributes.parquet', columns=['categories', 'characteristic_attributes_mapping'], engine='pyarrow')\n","attrs['category_level_2'] = attrs['categories'].progress_apply(lambda x: eval(x)['2'])"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["data = pd.concat([text_and_bert, attrs], axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["del text_and_bert, attrs\n","gc.collect()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def remove_html_tags_and_emoji(text):\n","    if text is None:\n","        return None\n","    clean = re.compile('<.*?>')\n","    text = re.sub(clean, '', text)\n","    text = text.replace('\\n', ' ')\n","    emoji_pattern = re.compile(\"[\"\n","                               u\"\\U0001F600-\\U0001F64F\"\n","                               u\"\\U0001F300-\\U0001F5FF\"\n","                               u\"\\U0001F680-\\U0001F6FF\"\n","                               u\"\\U0001F1E0-\\U0001F1FF\"\n","                               \"]+\", flags=re.UNICODE)\n","    return emoji_pattern.sub(r'', text)\n","\n","data['description'] = data['description'].progress_apply(remove_html_tags_and_emoji)\n","data['name'] = data['name'].progress_apply(remove_html_tags_and_emoji)\n","data['category_level_2'] = data['category_level_2'].progress_apply(lambda x: x.lower())"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["train_pairs = pd.read_parquet('/kaggle/input/extracted_data/train.parquet', engine='pyarrow')"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["train_pairs.rename(\n","    columns={\n","        'variantid1': 'variantid_1',\n","        'variantid2': 'variantid_2'\n","    }, inplace=True\n",")\n","\n","train_df = train_pairs.merge(\n","    data.add_suffix('_1'), \n","    on='variantid_1'\n",").merge(\n","    data.add_suffix('_2'), \n","    on='variantid_2'\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["train_df['category_level_2'] = train_df['category_level_2_1']\n","train_df.drop(columns=['category_level_2_1', 'category_level_2_2'], axis=1, inplace=True)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def get_dist_and_sim(dict1, dict2):\n","    dist, sim = [], []\n","    try:\n","        dict1, dict2 = eval(dict1), eval(dict2)\n","        dict_keys = set(dict1.keys()) & set(dict2.keys())\n","    except:\n","        return dist, sim\n","    \n","    for key in dict_keys:\n","        val1 = dict1.get(key)\n","        val2 = dict2.get(key)\n","        if val1 != val2:\n","            dist.append(key)\n","        if val1 == val2:\n","            sim.append(key)\n","    return dist, sim\n","\n","dataset = []\n","for i in tqdm(range(len(train_df))):\n","    row = train_df.iloc[i]\n","    target = row.target\n","    category = row.category_level_2\n","    name1 = row.name_1\n","    name2 = row.name_2\n","    desc1 = row.description_1\n","    desc2 = row.description_2\n","    res_dist, res_similar = get_dist_and_sim(\n","        row.characteristic_attributes_mapping_1,\n","        row.characteristic_attributes_mapping_2\n","    )\n","    dataset.append(\n","        (category, \n","         name1, \n","         name2,\n","         desc1, \n","         desc2, \n","         ', '.join(res_dist), \n","         ', '.join(res_similar), \n","         target)\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["model_name = 'cointegrated/rubert-tiny2'\n","\n","tokenizer = AutoTokenizer.from_pretrained(model_name)\n","model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2).cuda()\n","\n","batch_size = 32\n","optimizer = torch.optim.AdamW(model.parameters(), lr=3e-5)\n","epochs = 3\n","total_steps = (1 + len(dataset) // batch_size) * epochs\n","scheduler = get_linear_schedule_with_warmup(optimizer, \n","                                            num_warmup_steps=0, \n","                                            num_training_steps=total_steps)\n","\n","train_losses = []\n","for ep in range(epochs):\n","    np.random.shuffle(dataset)\n","    optimizer.zero_grad()\n","    losses = []\n","    pbar = tqdm(range(len(dataset)), desc=f'Epoch {ep} Loss 0.000', total=len(dataset))\n","    for t in pbar:\n","        category, name1, name2, desc1, desc2, dist, sim, target = dataset[t]\n","        name = name1 + '[SEP]' + name2\n","        desc_start = desc1[:300] + '[SEP]' + desc2[:300]\n","        desc_end = desc1[-300:] + '[SEP]' + desc2[-300:] \n","        s = category + '[SEP]' + 'Названия: ' + name + '[SEP]' + 'Описания: ' + desc_start + '[SEP]' + desc_end\n","        tks = tokenizer.encode_plus(\n","            s[:1800], \n","            max_length=1024, \n","            pad_to_max_length=False, \n","            return_attention_mask=True, \n","            return_tensors='pt', \n","            truncation=True\n","        )\n","        out = model(\n","            tks['input_ids'].cuda(), \n","            attention_mask=tks['attention_mask'].cuda(),\n","            token_type_ids=tks['token_type_ids'].cuda(),\n","            labels=torch.tensor([[1.0 - target, target]]).float().cuda()\n","        )\n","        \n","        losses.append(out.loss)\n","        \n","        if (t + 1) % batch_size == 0:\n","            loss = sum(losses) / batch_size\n","            loss.backward()\n","            losses = []\n","            train_losses.append(loss.item())\n","            optimizer.step() \n","            optimizer.zero_grad()\n","            scheduler.step()\n","            pbar.set_description(f'Epoch {ep} Loss {loss.item():.3f}')\n","    \n","    if len(losses) > 0:\n","        loss = sum(losses) / batch_size\n","        loss.backward()\n","        losses = []\n","        train_losses.append(loss.item())\n","        optimizer.step() \n","        optimizer.zero_grad()\n","        scheduler.step()\n","        pbar.set_description(f'Epoch {ep} Loss {loss.item():.3f}')\n","\n","torch.save(model.state_dict(), f'name_desc_bert_full.pth')"]}],"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"datasetId":5565627,"sourceId":9205012,"sourceType":"datasetVersion"},{"datasetId":5588610,"sourceId":9239049,"sourceType":"datasetVersion"}],"dockerImageVersionId":30762,"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"}},"nbformat":4,"nbformat_minor":4}