turkish-news-classification-using-han.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from wordcloud import WordCloud\n",
    "from collections import Counter\n",
    "\n",
    "import tqdm\n",
    "import nltk\n",
    "import re\n",
    "import io\n",
    "import string\n",
    "import emoji\n",
    "import gensim\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras import layers, models, optimizers, losses, regularizers, metrics, initializers, constraints\n",
    "from tensorflow.keras.utils import to_categorical, plot_model\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from mlxtend.plotting import plot_confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc, classification_report, confusion_matrix\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "from IPython.display import Markdown\n",
    "\n",
    "def bold(string):\n",
    "    display(Markdown(\"**\" + string + \"**\"))\n",
    "\n",
    "import warnings\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/kaggle/input/turkish-news-article/TurkishNewsArticles.csv\", usecols=[\"author\", \"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_stats(data):\n",
    "    bold(\" SHAPE \".center(50, \"#\"))\n",
    "    print(\"ROWS: {}\".format(data.shape[0]))\n",
    "    print(\"COLS: {}\".format(data.shape[1]))\n",
    "    bold(\" TYPES \".center(50, \"#\"))\n",
    "    print(data.dtypes)\n",
    "    bold(\" MISSING VALUES \".center(50, \"#\"))\n",
    "    print(data.isnull().sum())\n",
    "    bold(\" DUPLICATED VALUES \".center(50, \"#\"))\n",
    "    print(\"NUMBER OF DUPLICATED VALUES: {}\".format(data.duplicated().sum()))\n",
    "    #bold(\" DESCRIBE \".center(50, \"#\"))\n",
    "    #print(data.describe().T)\n",
    "    bold(\" MEMORY USAGE \".center(50, \"#\"))\n",
    "    buf = io.StringIO()\n",
    "    data.info(buf=buf)\n",
    "    info = buf.getvalue().split(\"\\n\")[-2].split(\":\")[1].strip()\n",
    "    print(\"Memory Usage: {}\".format(info))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_stats(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.dropna()\n",
    "df = df.drop_duplicates()\n",
    "df = df.sample(frac=1)\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"author\"].value_counts().head(15).plot(kind=\"pie\", autopct=\"%.1f%%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_counts = df['author'].value_counts()\n",
    "authors_to_keep = author_counts[author_counts >= 20].index\n",
    "df = df[df['author'].isin(authors_to_keep)]\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = Counter(df[\"author\"]).keys()\n",
    "tags_len = Counter(df[\"author\"]).values()\n",
    "tag_df = pd.DataFrame(zip(tags, tags_len), columns=[\"Class\", \"Count\"])\n",
    "tag_df.plot(x=\"Class\", y=\"Count\", kind=\"bar\", legend=False, grid=False, figsize=(30, 10), cmap='viridis')\n",
    "plt.title(\"Class / Count\", fontsize=18)\n",
    "plt.xlabel(\"Class\", fontsize=10)\n",
    "plt.ylabel(\"Count\", fontsize=15)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smart_words = \"/kaggle/input/turkish-news-article/stop-words.txt\"\n",
    "smart_stoplist = []\n",
    "for line in open(smart_words, \"r\"):\n",
    "    if line.strip()[0:1] != \"#\":\n",
    "        for word in line.split():\n",
    "            smart_stoplist.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"text\"] = [token.lower() for token in df[\"text\"]]\n",
    "#df[\"Text Cleaned\"] = df[\"Text Cleaned\"].apply(lambda x: re.sub('@[\\w_]+', '', x))\n",
    "#df[\"Text Cleaned\"] = df[\"Text Cleaned\"].apply(lambda x: re.sub('#[\\w_]+', '', x))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].replace(r'http\\S+', '', regex=True).replace(r'www\\S+', '', regex=True)\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: x.translate(x.maketrans('', '', string.punctuation)))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: x.replace('\"', '').replace(\"’\", '').replace(\"'\", '').replace(\"”\", ''))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: re.sub('\\S*@\\S*\\s?', '', x))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: emoji.replace_emoji(x))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: re.sub('<.*?>', '', x))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda x: re.sub('[0-9]+', '', x))\n",
    "#df['Text Cleaned'] = df['Text Cleaned'].apply(lambda text: ' '.join([word for word in text.split() if word.lower() not in smart_stoplist]))\n",
    "df[\"text\"] = df[\"text\"].apply(lambda x: re.sub(\"\\n\", \"\", x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_ngrams(corpus, ngram, n):\n",
    "    vec = CountVectorizer(ngram_range=(ngram,ngram)).fit(corpus)\n",
    "    bow = vec.transform(corpus).sum(axis=0)\n",
    "    words_freq = sorted([(word, bow[0, idx]) for word, idx in vec.vocabulary_.items()], key=lambda x: x[1], reverse=True)[:n]\n",
    "    return words_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_ngrams(ngram_df, ngram_name):\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    plt.bar(data=ngram_df, x=\"Text\", height=\"Count\")\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.xlabel(ngram_name)\n",
    "    plt.ylabel(\"Count\")\n",
    "    plt.title(ngram_name)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigrams = count_ngrams(df[\"text\"], 1, 30)\n",
    "top_unigram = pd.DataFrame(unigrams, columns=['Text', \"Count\"])\n",
    "top_unigram.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_ngrams(top_unigram, \"Unigrams\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigrams = count_ngrams(df[\"text\"], 2, 30)\n",
    "top_bigram = pd.DataFrame(bigrams, columns=['Text', \"Count\"])\n",
    "top_bigram.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_ngrams(top_bigram, \"Bigrams\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigrams = count_ngrams(df[\"text\"], 3, 30)\n",
    "top_trigram = pd.DataFrame(trigrams, columns=['Text', \"Count\"])\n",
    "top_trigram.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_ngrams(top_trigram, \"Trigrams\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "df[\"author\"] = le.fit_transform(df[\"author\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "authors = df[\"author\"]\n",
    "text = df[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 20000\n",
    "texts = text.tolist()\n",
    "tokenizer = Tokenizer(num_words=max_features, filters = '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', oov_token=True)\n",
    "tokenizer.fit_on_texts(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = [sent_tokenize(text) for text in texts]\n",
    "\n",
    "max_sentence_len = 15\n",
    "max_sentence_num = 35\n",
    "embed_size = 100\n",
    "\n",
    "data = np.zeros((len(texts), max_sentence_num, max_sentence_len), dtype='int32')\n",
    "\n",
    "for i, sentences in enumerate(params):\n",
    "    for j, sent in enumerate(sentences[:max_sentence_num]):\n",
    "        word_tokens = text_to_word_sequence(sent)\n",
    "        word_indices = [tokenizer.word_index[word] for word in word_tokens if word in tokenizer.word_index and tokenizer.word_index[word] < max_features]\n",
    "        data[i, j, :len(word_indices[:max_sentence_len])] = word_indices[:max_sentence_len]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = to_categorical(authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, labels, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_train shape:\", X_train.shape)\n",
    "print(\"y_train shape:\", y_train.shape)\n",
    "print(\"X_test shape:\", X_test.shape)\n",
    "print(\"y_test shape:\", y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_index = {}\n",
    "f = open(\"/kaggle/input/glove-global-vectors-for-word-representation/glove.twitter.27B.100d.txt\", \"r\", encoding=\"utf-8\")\n",
    "for line in tqdm.tqdm(f):\n",
    "    values = line.split(\" \")\n",
    "    word = values[0]\n",
    "    coefs = np.asarray([float(val) for val in values[1:]])\n",
    "    embeddings_index[word] = coefs\n",
    "\n",
    "f.close()\n",
    "print(\"Found %s word vectors.\" % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim = len(tokenizer.word_index) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.zeros((input_dim, 100))\n",
    "for word, i in tqdm.tqdm(tokenizer.word_index.items()):\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttentionWithContext(layers.Layer):\n",
    "    def __init__(self,\n",
    "                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, u_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.u_regularizer = regularizers.get(u_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.u_constraint = constraints.get(u_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        super(AttentionWithContext, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight(shape=(input_shape[-1], input_shape[-1]),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight(shape=(input_shape[-1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "\n",
    "        self.u = self.add_weight(shape=(input_shape[-1], 1),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_u'.format(self.name),\n",
    "                                 regularizer=self.u_regularizer,\n",
    "                                 constraint=self.u_constraint)\n",
    "\n",
    "        super(AttentionWithContext, self).build(input_shape)\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        uit = K.dot(x, self.W)\n",
    "\n",
    "        if self.bias:\n",
    "            uit += self.b\n",
    "            \n",
    "        uit = K.tanh(uit)\n",
    "        ait = K.squeeze(K.dot(uit, self.u), -1)\n",
    "        a = K.exp(ait)\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0], input_shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_input = layers.Input(shape=(max_sentence_len,), dtype='float32')\n",
    "x = layers.Embedding(len(tokenizer.word_index) + 1, \n",
    "                     embed_size, \n",
    "                     weights=[embedding_matrix], \n",
    "                     input_length=max_sentence_len, \n",
    "                     trainable=False)(word_input)\n",
    "x = layers.Bidirectional(layers.LSTM(128, return_sequences=True, kernel_regularizer=\"l2\"))(x)\n",
    "x = layers.TimeDistributed(layers.Dense(256, kernel_regularizer=\"l2\"))(x)\n",
    "x = AttentionWithContext()(x)\n",
    "word_encoder = models.Model(word_input, x)\n",
    "\n",
    "sentence_input = layers.Input(shape=(max_sentence_num, max_sentence_len), dtype='float32')\n",
    "x = layers.TimeDistributed(word_encoder)(sentence_input)\n",
    "x = layers.Bidirectional(layers.LSTM(128, return_sequences=True, kernel_regularizer=\"l2\"))(x)\n",
    "x = layers.TimeDistributed(layers.Dense(256, kernel_regularizer=\"l2\"))(x)\n",
    "x = layers.Dropout(0.5)(AttentionWithContext()(x))\n",
    "output_layer = layers.Dense(authors.nunique(), activation='softmax')(x)\n",
    "\n",
    "model = models.Model(sentence_input, output_layer)\n",
    "model.compile(loss='categorical_crossentropy', \n",
    "              optimizer='adam', \n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.utils.plot_model(model, expand_nested=True, show_layer_names=True, show_shapes=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#early_stopping = EarlyStopping(monitor='val_accuracy', mode='max', patience=5)\n",
    "#lr_scheduler = ReduceLROnPlateau(monitor='val_accuracy', mode='max', patience=5, factor=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(X_train, y_train, validation_split=0.1, epochs=100, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss, test_acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(f\"Test Loss: {test_loss:.5f}\")\n",
    "print(f\"Test Accuracy: {test_acc:.5f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history_df = pd.DataFrame(history.history)\n",
    "history_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(history.history[\"loss\"])\n",
    "plt.plot(history.history[\"val_loss\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend([\"train\", \"valid\"])\n",
    "plt.title(\"Loss Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(history.history[\"accuracy\"])\n",
    "plt.plot(history.history[\"val_accuracy\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend([\"train\", \"valid\"])\n",
    "plt.title(\"Accuracy Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}