notebooks/1.1 Preprocessing & Feature Generation for Train.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2024-08-24T10:56:37.448760Z","iopub.status.busy":"2024-08-24T10:56:37.447646Z"},"scrolled":true,"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting rapidfuzz\n","  Downloading rapidfuzz-3.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n","Collecting jellyfish\n","  Downloading jellyfish-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.6 kB)\n","Collecting textdistance\n","  Downloading textdistance-4.6.3-py3-none-any.whl.metadata (18 kB)\n","Downloading rapidfuzz-3.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n","\u001b[?25hDownloading jellyfish-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (335 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m336.0/336.0 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading textdistance-4.6.3-py3-none-any.whl (31 kB)\n"]}],"source":["!pip install rapidfuzz jellyfish textdistance"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import json\n","import ast\n","import pickle\n","import gc\n","import re\n","\n","from rapidfuzz import fuzz\n","import jellyfish\n","import textdistance\n","\n","from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n","from sklearn.feature_extraction.text import TfidfVectorizer\n","\n","import scipy.sparse as sp\n","\n","from tqdm import tqdm\n","tqdm.pandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["attrs = pd.read_parquet('/kaggle/input/extracted_data/attributes.parquet', engine='pyarrow')\n","resnet = pd.read_parquet('/kaggle/input/extracted_data/resnet.parquet', engine='pyarrow')\n","text_and_bert = pd.read_parquet('/kaggle/input/extracted_data/text_and_bert.parquet', engine='pyarrow')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["train_pairs = pd.read_parquet('/kaggle/input/extracted_data/train.parquet', engine='pyarrow')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["data = pd.concat([attrs, resnet.drop(columns=['variantid']), text_and_bert.drop(columns=['variantid'])], axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["data['variantid'] = data['variantid'].astype('uint32')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["del attrs, resnet, text_and_bert\n","gc.collect()"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["data['description'] = data['description'].fillna('no desc')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Нормализация текста\n","def normalize(text):\n","    if text is None:\n","        return None\n","    text = text.lower()\n","    chars = []\n","    for char in text:\n","        if char.isalnum():\n","            chars.append(char)\n","        else:\n","            chars.append(' ')\n","    tokens = ''.join(chars).split() \n","    return '_'.join(tokens)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Удаление html тэгов и эмодзи из строки\n","def remove_html_tags_and_emoji(text):\n","    if text is None:\n","        return None\n","    clean = re.compile('<.*?>')\n","    text = re.sub(clean, '', text)\n","    text = text.replace('\\n', ' ')\n","    emoji_pattern = re.compile(\"[\"\n","                               u\"\\U0001F600-\\U0001F64F\"\n","                               u\"\\U0001F300-\\U0001F5FF\"\n","                               u\"\\U0001F680-\\U0001F6FF\"\n","                               u\"\\U0001F1E0-\\U0001F1FF\"\n","                               \"]+\", flags=re.UNICODE)\n","    return emoji_pattern.sub(r'', text)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Нормализация названий товаров\n","def normalize_names(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Нормализую названия товаров...')\n","    df['name'] = df['name'].progress_apply(remove_html_tags_and_emoji)\n","    df['name_norm'] = df['name'].progress_apply(normalize)\n","    df['name_tokens'] = df['name'].str.strip().str.lower()\n","    df['name'] = df['name_tokens'].progress_apply(lambda tokens: ' '.join(tokens.split()))\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Нормализация описаний товаров\n","def normalize_desc(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Нормализую описания товаров...')\n","    df['description'] = df['description'].progress_apply(remove_html_tags_and_emoji)\n","    df['description_norm'] = df['description'].progress_apply(normalize)\n","    df['description_tokens'] = df['description'].str.strip().str.lower()\n","    df['description'] = df['description_tokens'].progress_apply(lambda tokens: ' '.join(tokens.split()))\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Выделение бренда как отдельной фичи\n","def extract_brand(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Извлекаю названия брендов...')\n","    brand_arr = []\n","    for i in tqdm(range(len(df))):\n","        try:\n","            brand_arr.append(json.loads(df['characteristic_attributes_mapping'][i])['Бренд'][0]) \n","        except:\n","            brand_arr.append(None)\n","\n","    df['brand'] = brand_arr\n","\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Выделение страны как отдельной фичи\n","def extract_country(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Извлекаю страны-изготовители...')\n","    country_arr = []\n","    for i in tqdm(range(len(df))):\n","        try:\n","            country_arr.append(json.loads(df['characteristic_attributes_mapping'][i])['Страна-изготовитель'][0]) \n","        except:\n","            country_arr.append(None)\n","\n","    df['country'] = country_arr\n","\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Извлечение категорий\n","def extract_categories(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Извлекаю категории...')\n","    categories = pd.json_normalize(df['categories'].progress_apply(ast.literal_eval))\n","    categories.columns = [f'category_level_{i+1}' for i in range(categories.shape[1])]\n","    return df.drop(columns=['categories']).join(categories)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# [[emb]] -> [emb]\n","def squeeze_main_pic_embeddings(df: pd.DataFrame) -> pd.DataFrame:\n","    print('Распаковываю эмбеддинги...')\n","    df['main_pic_embeddings_resnet_v1'] = df['main_pic_embeddings_resnet_v1'].progress_apply(\n","        lambda x: x[0] if isinstance(x, np.ndarray) else x\n","    )\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Нормализация атрибутов\n","def normalize_characteristic_attributes(df: pd.DataFrame) -> pd.DataFrame:\n","    def normalize_attributes(char_attrs_map):\n","        if char_attrs_map is not None:\n","            char_attrs_map = ast.literal_eval(char_attrs_map)\n","            parsed = [normalize(key) for key in char_attrs_map.keys()]\n","            return '; '.join([''.join(val) for val in parsed])\n","        return 'none'\n","\n","    def normalize_values(attrs_map):\n","        if attrs_map is not None:\n","            attrs_map = ast.literal_eval(attrs_map)\n","            parsed = [list(map(normalize, attr_list)) for attr_list in attrs_map.values()]\n","            return '; '.join([' '.join(val) for val in parsed])\n","        return 'none'\n","\n","    print('Нормализую значения атрибутов...')\n","    df['attr_vals'] = df['characteristic_attributes_mapping'].progress_apply(normalize_values)\n","    \n","    print('Нормализую атрибуты...')\n","    df['attr_keys'] = df['characteristic_attributes_mapping'].progress_apply(normalize_attributes)\n","    \n","    def combine_char_attributes(dct):\n","        if dct is not None:\n","            parsed = ast.literal_eval(dct)\n","            return '; '.join([f'{k}:{v}' for k, v in parsed.items()])\n","        return 'none'\n","    \n","    print('Собираю атрибуты и значения...')\n","    df['characteristics_attributes'] = df['characteristic_attributes_mapping'].progress_apply(combine_char_attributes)\n","    \n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Количество картинок, токенов в названии, в описании\n","def get_lengths(df: pd.DataFrame) -> pd.DataFrame:\n","    len_w_nans = lambda x: len(x) if x is not None else None\n","    \n","    print('Создаю количественные фичи...')\n","    df['pic_embeddings_resnet_v1_len'] = df['pic_embeddings_resnet_v1'].progress_apply(len_w_nans)\n","    df['name_tokens_len'] = df['name_tokens'].apply(lambda x: x.split()).progress_apply(len_w_nans)\n","    df['description_tokens_len'] = df['description_tokens'].apply(lambda x: x.split()).progress_apply(len_w_nans)\n","    df['characteristics_attributes_len'] = df['characteristics_attributes'].apply(lambda x: x.split('; ')).progress_apply(len_w_nans)\n","    \n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Извлечение чисел из строк\n","def get_digits_elements(df: pd.DataFrame) -> pd.DataFrame:\n","    def has_more_than_two_digits(s):\n","        return len(re.findall(r'\\d', s)) > 2\n","\n","    print('Нахожу числа в названиях, описаниях и атрибутах...')\n","    for col in ('attr_vals', 'name_tokens', 'description_tokens'):\n","        if 'attr' not in col:\n","            df[f'{col}_w_digits'] = df[col].progress_apply(lambda row: ' '.join([s for s in row.split() if has_more_than_two_digits(s)]))\n","        else:\n","            df[f'{col}_w_digits'] = df[col].progress_apply(lambda row: ' '.join([s for s in row.split('; ') if has_more_than_two_digits(s)]))\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Конкатенированный эмбеддинг bert и resnet\n","def concat_embs(df: pd.DataFrame) -> pd.DataFrame:\n","    def normalize(array):\n","        norm = np.linalg.norm(array)\n","        if norm == 0:\n","            return array\n","        return array / norm\n","    \n","    print('Конкатенирую эмбеддинги...')\n","    df['concat_emb'] = df.progress_apply(\n","        lambda row: np.concatenate(\n","            [\n","                normalize(row['main_pic_embeddings_resnet_v1']), \n","                normalize(row['name_bert_64'])\n","            ]\n","        ), \n","        axis=1\n","    )\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def fit_tfidf_vectorizer(data, columns):\n","    tfidf_vectorizers = {}\n","    for col in columns:\n","        vectorizer = TfidfVectorizer()\n","        combined_texts = data[col].astype(str).tolist()\n","        vectorizer.fit(combined_texts)\n","        tfidf_vectorizers[col] = vectorizer\n","        \n","        with open(f'{col}_tfidf_vectorizer.pkl', 'wb') as f:\n","            pickle.dump(vectorizer, f)\n","            \n","    return tfidf_vectorizers\n","\n","columns = ['name', 'description', 'attr_keys', 'attr_vals']"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Tfidf фичи\n","def tfidf_emb_gen(data, tfidf_vectorizers, columns, batch_size=5000):\n","    for col in columns:\n","        tfidf_col_sparse = []\n","        for start in tqdm(range(0, len(data), batch_size)):\n","            end = min(start + batch_size, len(data))\n","            batch_texts = data[col].iloc[start:end].astype(str).tolist()\n","            tfidf_batch_sparse = tfidf_vectorizers[col].transform(batch_texts)\n","            tfidf_col_sparse.append(tfidf_batch_sparse)\n","        tfidf_col_sparse = sp.vstack(tfidf_col_sparse)\n","        data[f'{col}_tfidf'] = [row for row in tfidf_col_sparse]\n","    return data"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def preprocess(data: pd.DataFrame) -> pd.DataFrame:\n","    data = extract_categories(data)\n","    data = normalize_names(data)\n","    data = normalize_desc(data)\n","    data = extract_brand(data)\n","    data = extract_country(data)\n","    data = squeeze_main_pic_embeddings(data)\n","    data = normalize_characteristic_attributes(data)\n","    data = get_lengths(data)\n","    data = get_digits_elements(data)\n","    data = concat_embs(data)\n","    data = tfidf_emb_gen(data, tfidf_vectorizers=fit_tfidf_vectorizer(data, columns), columns=columns)\n","    return data"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["data = preprocess(data)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["train_pairs.rename(\n","    columns={\n","        'variantid1': 'variantid_1',\n","        'variantid2': 'variantid_2'\n","    }, inplace=True\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["df = train_pairs.merge(\n","    data.add_suffix('_1'), \n","    on='variantid_1'\n",").merge(\n","    data.add_suffix('_2'), \n","    on='variantid_2'\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по категориям (полное совпадение + частичное по последнему уровню)\n","for i in range(1, 5):\n","    df[f'category_level_{i}_match'] = df.progress_apply(\n","        lambda row: row[f'category_level_{i}_1'].lower() == row[f'category_level_{i}_2'].lower(), axis=1\n","    )\n","    if i == 4:\n","        df[f'category_level_{i}_token_sort_ratio_match'] = df.progress_apply(\n","            lambda row: fuzz.token_sort_ratio(row[f'category_level_{i}_1'], row[f'category_level_{i}_2']) / 100, axis=1\n","        )"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Мэтч по описанию\n","df[f'description_match'] = df.progress_apply(lambda row: row['description_1'] == row['description_2'], axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def match_brends(row):\n","    if row[f'brand_1'] is None or row[f'brand_2'] is None:\n","        return None\n","    return row[f'brand_1'].lower() == row[f'brand_2'].lower()\n","\n","# Мэтч по бренду   \n","df[f'brand_match'] = df.progress_apply(match_brends, axis=1)\n","df.drop(columns=['brand_1', 'brand_2'], axis=1, inplace=True)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def match_countries(row):\n","    if row[f'country_1'] is None or row[f'country_2'] is None:\n","        return None\n","    return row[f'country_1'].lower() == row[f'country_2'].lower()\n","\n","# Мэтч по стране\n","df[f'country_match'] = df.progress_apply(match_countries, axis=1)\n","df.drop(columns=['country_1', 'country_2'], axis=1, inplace=True)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# Мэтч по описанию  \n","df[f'description_match'] = df.progress_apply(lambda row: row[f'description_1'] == row[f'description_2'], axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Отношения длин\n","for col in ('pic_embeddings_resnet_v1_len', 'name_tokens_len', 'description_tokens_len', 'characteristics_attributes_len'):\n","    df[f'{col}_ratio_left'] = df.progress_apply(\n","        lambda row: row[f'{col}_1'] / row[f'{col}_2'] if row[f'{col}_2'] not in (0, None) else 0, axis=1\n","    )\n","    df[f'{col}_ratio_right'] = df.progress_apply(\n","        lambda row: row[f'{col}_2'] / row[f'{col}_1'] if row[f'{col}_1'] not in (0, None) else 0, axis=1\n","    )\n","    \n","for col in ('attr_vals', 'attr_vals_w_digits', 'attr_keys', 'name_tokens_w_digits'):\n","    df[f'{col}_ratio_left'] = df.progress_apply(\n","        lambda row: len(row[f'{col}_1'].split()) / len(row[f'{col}_2'].split()) if len(row[f'{col}_2'].split()) not in (0, None) else 0, axis=1\n","    )\n","    df[f'{col}_ratio_right'] = df.progress_apply(\n","        lambda row: len(row[f'{col}_2'].split()) / len(row[f'{col}_1'].split()) if len(row[f'{col}_1'].split()) not in (0, None) else 0, axis=1\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def cosine_sim(vec1, vec2):\n","    vec1 = vec1.reshape(1, -1)\n","    vec2 = vec2.reshape(1, -1)\n","    return cosine_similarity(vec1, vec2)[0][0]\n","\n","def euc_dist(vec1, vec2):\n","    vec1 = vec1.reshape(1, -1)\n","    vec2 = vec2.reshape(1, -1)\n","    return euclidean_distances(vec1, vec2)[0][0]"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по main_pic_embeddings_resnet_v1 с main_pic_embeddings_resnet_v1\n","df['main_pic_embeddings_resnet_v1_cos_sim'] = df.progress_apply(\n","    lambda row: cosine_sim(row['main_pic_embeddings_resnet_v1_1'], row['main_pic_embeddings_resnet_v1_2']), axis=1\n",")\n","df['main_pic_embeddings_resnet_v1_euc_dist'] = df.progress_apply(\n","    lambda row: euc_dist(row['main_pic_embeddings_resnet_v1_1'], row['main_pic_embeddings_resnet_v1_2']), axis=1\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def pair_cos_sim(vecs1, vecs2):\n","    if vecs1 is None or vecs2 is None:\n","        return {\n","            'mean': None,\n","            'median': None,\n","            'min': None,\n","            'max': None,\n","            'std': None\n","        }\n","    \n","    sim = []\n","    for vec1 in vecs1:\n","        for vec2 in vecs2:\n","            sim.append(cosine_sim(vec1, vec2))\n","    \n","    sim_array = np.array(sim)\n","    return {\n","        'mean': np.mean(sim_array),\n","        'median': np.median(sim_array),\n","        'min': np.min(sim_array),\n","        'max': np.max(sim_array),\n","        'std': np.std(sim_array)\n","    }\n","\n","# Мэтч по pic_embeddings_resnet_v1 с pic_embeddings_resnet_v1\n","results = df.progress_apply(\n","    lambda row: pair_cos_sim(row['pic_embeddings_resnet_v1_1'], row['pic_embeddings_resnet_v1_2']), axis=1\n",")\n","df['pic_embeddings_resnet_v1_mean_cos_sim'] = results.apply(lambda x: x['mean'])\n","df['pic_embeddings_resnet_v1_median_cos_sim'] = results.apply(lambda x: x['median'])\n","df['pic_embeddings_resnet_v1_min_cos_sim'] = results.apply(lambda x: x['min'])\n","df['pic_embeddings_resnet_v1_max_cos_sim'] = results.apply(lambda x: x['max'])\n","df['pic_embeddings_resnet_v1_std_cos_sim'] = results.apply(lambda x: x['std'])"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def cross_cos_sim(vec1, vecs2):\n","    if vec1 is None or vecs2 is None:\n","        return {\n","            'mean': None,\n","            'median': None,\n","            'min': None,\n","            'max': None,\n","            'std': None\n","        }\n","    sim = []\n","    for vec2 in vecs2:\n","        sim.append(cosine_sim(vec1, vec2))\n","    \n","    sim_array = np.array(sim)\n","    return {\n","        'mean': np.mean(sim_array),\n","        'median': np.median(sim_array),\n","        'min': np.min(sim_array),\n","        'max': np.max(sim_array),\n","        'std': np.std(sim_array)\n","    }\n","\n","# Мэтч по main_pic_embeddings_resnet_v1 с pic_embeddings_resnet_v1\n","results = df.progress_apply(\n","    lambda row: cross_cos_sim(row['main_pic_embeddings_resnet_v1_1'], row['pic_embeddings_resnet_v1_2']), axis=1\n",")\n","df['cross1_mean_cos_sim'] = results.apply(lambda x: x['mean'])\n","df['cross1_median_cos_sim'] = results.apply(lambda x: x['median'])\n","df['cross1_min_cos_sim'] = results.apply(lambda x: x['min'])\n","df['cross1_max_cos_sim'] = results.apply(lambda x: x['max'])\n","df['cross1_std_cos_sim'] = results.apply(lambda x: x['std'])\n","\n","results = df.progress_apply(\n","    lambda row: cross_cos_sim(row['main_pic_embeddings_resnet_v1_2'], row['pic_embeddings_resnet_v1_1']), axis=1\n",")\n","df['cross2_mean_cos_sim'] = results.apply(lambda x: x['mean'])\n","df['cross2_median_cos_sim'] = results.apply(lambda x: x['median'])\n","df['cross2_min_cos_sim'] = results.apply(lambda x: x['min'])\n","df['cross2_max_cos_sim'] = results.apply(lambda x: x['max'])\n","df['cross2_std_cos_sim'] = results.apply(lambda x: x['std'])"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по bert\n","df['name_bert_64_cos_sim'] = df.progress_apply(\n","    lambda row: cosine_sim(row['name_bert_64_1'], row['name_bert_64_2']), axis=1\n",")\n","df['name_bert_64_euc_dist'] = df.progress_apply(\n","    lambda row: euc_dist(row['name_bert_64_1'], row['name_bert_64_2']), axis=1\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по concat_emb\n","df['concat_emb_cos_sim'] = df.progress_apply(\n","    lambda row: cosine_sim(row['concat_emb_1'], row['concat_emb_2']), axis=1\n",")\n","df['concat_emb_euc_dist'] = df.progress_apply(\n","    lambda row: euc_dist(row['concat_emb_1'], row['concat_emb_2']), axis=1\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по tfidf\n","for col in ('name', 'description', 'attr_keys', 'attr_vals'):\n","    df[f'{col}_tfidf_cos_sim'] = df.progress_apply(\n","        lambda row: cosine_sim(row[f'{col}_tfidf_1'], row[f'{col}_tfidf_2']), axis=1\n","    )\n","    df[f'{col}_tfidf_euc_dist'] = df.progress_apply(\n","        lambda row: euc_dist(row[f'{col}_tfidf_1'], row[f'{col}_tfidf_2']), axis=1\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def fillness(df: pd.DataFrame, col_name: str) -> pd.DataFrame:\n","    condition_both = df[f'{col_name}_1'].notna() & df[f'{col_name}_2'].notna()\n","    condition_none = df[f'{col_name}_1'].isna() & df[f'{col_name}_2'].isna()\n","    \n","    df[f'{col_name}_fillness'] = np.where(\n","        condition_both, 'both',\n","        np.where(condition_none, 'none', 'only one')\n","    )\n","    \n","    return df\n","\n","# Заполненность строк у товаров\n","df = fillness(df, 'main_pic_embeddings_resnet_v1')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def avg_fully_eq_attributes(d1, d2):\n","    if d1 is None or d2 is None:\n","        return None\n","    d1 = ast.literal_eval(d1)\n","    d2 = ast.literal_eval(d2)\n","    keys = set(d1) & set(d2)\n","    metrics = []\n","    for key in keys:\n","        metrics.append(set(d1[key]) == set(d2[key]))\n","    return np.mean(metrics)\n","\n","# Совпадения для словаря атрибутов\n","df['attributes_values_avg_fully_eq'] = (\n","    df.progress_apply(\n","        lambda row: avg_fully_eq_attributes(\n","            row['characteristic_attributes_mapping_1'],\n","            row['characteristic_attributes_mapping_2'],\n","        ),\n","        axis=1\n","    )\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Мэтч по названию и атрибутам\n","for col in ('name', 'name_norm', 'attr_vals', 'attr_keys', 'characteristics_attributes'):\n","    df[f'{col}_token_sort_ratio'] = df.progress_apply(\n","        lambda row: fuzz.token_sort_ratio(row[f'{col}_1'], row[f'{col}_2']) / 100, axis=1\n","    )\n","    df[f'{col}_token_set_ratio'] = df.progress_apply(\n","        lambda row: fuzz.token_set_ratio(row[f'{col}_1'], row[f'{col}_2']) / 100, axis=1\n","    )\n","    df[f'{col}_jaro_winkler_similarity'] = df.progress_apply(\n","        lambda row: jellyfish.jaro_winkler_similarity(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )\n","    df[f'{col}_dice'] = df.progress_apply(\n","        lambda row: textdistance.dice(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )\n","    df[f'{col}_tanimoto'] = df.progress_apply(\n","        lambda row: textdistance.tanimoto(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )\n","    df[f'{col}_sorensen'] = df.progress_apply(\n","        lambda row: textdistance.sorensen(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )\n","    if 'attr' not in col:\n","        df[f'{col}_damerau_levenshtein_distance'] = df.progress_apply(\n","            lambda row: jellyfish.damerau_levenshtein_distance(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","        )\n","        df[f'{col}_WRatio'] = df.progress_apply(\n","            lambda row: fuzz.WRatio(row[f'{col}_1'], row[f'{col}_2']) / 100, axis=1\n","        )\n","\n","# Мэтч по описанию    \n","for col in ('description',):\n","    df[f'{col}_jaro_winkler_similarity'] = df.progress_apply(\n","        lambda row: jellyfish.jaro_winkler_similarity(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def longest_common_prefix(str1, str2):\n","    if str1 is None or str2 is None:\n","        return None\n","    \n","    min_len = min(len(str1), len(str2))\n","    prefix_len = 0\n","    \n","    for i in range(min_len):\n","        if str1[i] == str2[i]:\n","            prefix_len += 1\n","        else:\n","            break\n","    \n","    return prefix_len / min_len if min_len != 0 else 0\n","\n","def longest_common_subsequence(str1, str2):\n","    if str1 is None or str2 is None:\n","        return None\n","    \n","    len1, len2 = len(str1), len(str2)\n","    dp = [[0] * (len2 + 1) for _ in range(len1 + 1)]\n","    \n","    for i in range(1, len1 + 1):\n","        for j in range(1, len2 + 1):\n","            if str1[i - 1] == str2[j - 1]:\n","                dp[i][j] = dp[i - 1][j - 1] + 1\n","            else:\n","                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])\n","    \n","    lcs_len = dp[len1][len2]\n","    return lcs_len / max(len1, len2) if max(len1, len2) != 0 else 0"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# LCP + LCS для названий\n","for col in ('name_norm',):\n","    df[f'{col}_lcp'] = df.progress_apply(\n","        lambda row: longest_common_prefix(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )\n","    df[f'{col}_lcs'] = df.progress_apply(\n","        lambda row: longest_common_subsequence(row[f'{col}_1'], row[f'{col}_2']), axis=1\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["def jaccard_similarity(list1, list2):\n","    if list1 is None or list2 is None:\n","        return None\n","    set1 = set(list1)\n","    set2 = set(list2)\n","    intersection = len(set1.intersection(set2))\n","    union = len(set1.union(set2))\n","    return intersection / union if union != 0 else 0\n","\n","def overlap_coefficient(list1, list2):\n","    if list1 is None or list2 is None:\n","        return None\n","    set1 = set(list1)\n","    set2 = set(list2)\n","    intersection = len(set1.intersection(set2))\n","    return intersection / min(len(set1), len(set2)) if min(len(set1), len(set2)) != 0 else 0"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Сходство для списков\n","for col in (\n","    'attr_keys', \n","    'attr_vals', \n","    'description_tokens', \n","    'name_tokens', \n","    'attr_vals_w_digits', \n","    'description_tokens_w_digits', \n","    'name_tokens_w_digits'\n","):\n","    df[f'{col}_jaccard_score'] = df.progress_apply(\n","        lambda row: jaccard_similarity(row[f'{col}_1'].split(), row[f'{col}_2'].split()), axis=1\n","    )\n","    df[f'{col}_overlap_score'] = df.progress_apply(\n","        lambda row: overlap_coefficient(row[f'{col}_1'].split(), row[f'{col}_2'].split()), axis=1\n","    )"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# Удалим ненужные фичи\n","df.drop(\n","    columns=[\n","        'name_1', 'name_2', \n","        'description_1', 'description_2', \n","        'name_norm_1', 'name_norm_2',\n","        'description_norm_1', 'description_norm_2', \n","        'attr_vals_1', 'attr_vals_2',\n","        'attr_keys_1', 'attr_keys_2',\n","        'characteristics_attributes_1', 'characteristics_attributes_2',\n","        'description_tokens_1', 'description_tokens_2',\n","        'name_tokens_1', 'name_tokens_2', \n","        'attr_vals_w_digits_1', 'attr_vals_w_digits_2', \n","        'description_tokens_w_digits_1', 'description_tokens_w_digits_2',\n","        'name_tokens_w_digits_1', 'name_tokens_w_digits_2',\n","        'name_tfidf_1', 'name_tfidf_2',\n","        'description_tfidf_1', 'description_tfidf_2',\n","        'attr_keys_tfidf_1', 'attr_keys_tfidf_2',\n","        'attr_vals_tfidf_1', 'attr_vals_tfidf_2', \n","        'name_bert_64_1', 'name_bert_64_2', \n","        'main_pic_embeddings_resnet_v1_1', 'main_pic_embeddings_resnet_v1_2', \n","        'pic_embeddings_resnet_v1_1', 'pic_embeddings_resnet_v1_2',\n","        'concat_emb_1', 'concat_emb_2',\n","    ], \n","    axis=1, \n","    inplace=True\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["df.to_parquet('train.parquet', index=False)"]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"datasetId":3324098,"sourceId":5785473,"sourceType":"datasetVersion"},{"datasetId":5565627,"sourceId":9205012,"sourceType":"datasetVersion"}],"dockerImageVersionId":30761,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"}},"nbformat":4,"nbformat_minor":4}