notebooks/add_memotion_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3c7M74FfxTlu"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "annotations_path = f\"hateful_memes/data/\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip memotion.zip"
      ],
      "metadata": {
        "id": "6JOddTqgxb8t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labeled_memo_samples = pd.read_json(f\"{annotations_path}/label_memotion.jsonl\", lines=True)['img']\n",
        "labeled_memo_samples = [i.split('/')[1] for i in list(labeled_memo_samples)]\n",
        "\n",
        "img_dir = f\"memotion_dataset_7k/images/\"\n",
        "for img in labeled_memo_samples:\n",
        "    os.rename(f\"{img_dir+img}\", f\"hateful_memes/data/img/{img}\")"
      ],
      "metadata": {
        "id": "Kzblg2sdxfJB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls hateful_memes/data/img/ | wc -l"
      ],
      "metadata": {
        "id": "FZL2Eb6ixfBM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "memotion = pd.read_json(f\"{annotations_path}/label_memotion.jsonl\", lines=True)\n",
        "train = pd.concat([train_updated, memotion], axis=0)\n",
        "# Shuffle data\n",
        "train = train.sample(frac=1).reset_index(drop=True)\n",
        "# Write new jsonl file\n",
        "train_json = train.to_json(orient='records', lines=True)\n",
        "\n",
        "with open(f\"{annotations_path}/train_updated.jsonl\", \"w\", encoding='utf-8') as f:\n",
        "    f.write(train_json)"
      ],
      "metadata": {
        "id": "zhH9jy9kxe7l"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}