guildedchatexporter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Fatih120/GuildedChatExporter/blob/main/guildedchatexporter.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# GuildedChatExporter\n",
        "\n",
        "Export your stuff before it's gone.\n",
        "\n",
        "## Heavy WIP. Works on channels with text, so text channels, even voice and streaming channels.\n",
        "\n",
        "1.   Set up a bot within the server you want to export from and create an Authentication Key for it. Paste it in the first box below.\n",
        "2.   Grab the Channel ID of the channel you want to archive. Have Settings > Advanced > Developer mode enabled, then right click a channel and copy the ID: or, find the ID in the channel URL after /channel/*. Paste it below.\n",
        "3.   Run the cells one by one, skipping the optional ones since you likely don't need them. Do this by clicking the play circle on the top left. Keep your Google Drive open to see your files.\n",
        "\n",
        "Need help?\n",
        "https://github.com/Fatih120/GuildedChatExporter/tree/main\n",
        "\n",
        "\n",
        "[Mountain of Fatih Guilded](https://guilded.gg/MoF)\n",
        "\n",
        "[Discord :(](https://discord.com/invite/Cy27FNfQtc)\n",
        "\n"
      ],
      "metadata": {
        "id": "F1WCwpFFOpC9"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eKEPJjYptE2I",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "# @title Basic setup\n",
        "API_KEY = \"gapi_KEY==\" # @param {type:\"string\"}\n",
        "SERVER_ID = \"\" # @param {type:\"string\"}\n",
        "CHANNEL_ID = \"XXXXXXXX-YYYY-ZZZZ-WWWW-YYYYYYYYYYYY \" # @param {type:\"string\"}\n",
        "SAVE_DIRECTORY = \"/guildedchatexporter/\" # @param {type:\"string\"}\n",
        "# @markdown Google Colab gives you over 100GB of temporary space to work with, which should be way more than enough to store and then download your stuff.\n",
        "# @markdown\n",
        "# @markdown  You can also connect your Google Drive instead and copy the stuff to there at the end, though if you just want to download your files immediately without using Google Drive, skip the optional step.\n",
        "import requests\n",
        "import json\n",
        "import os\n",
        "import re\n",
        "\n",
        "headers = {\n",
        "    'Authorization': f'Bearer {API_KEY}',\n",
        "    'Accept': 'application/json',\n",
        "    'Content-Type': 'application/json'\n",
        "}\n",
        "if not os.path.exists(SAVE_DIRECTORY):\n",
        "  os.makedirs(SAVE_DIRECTORY)\n",
        "SERVER_URL = f'https://www.guilded.gg/api/v1/servers/{SERVER_ID}'\n",
        "CHANNEL_MSG = f'https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}/messages'\n",
        "print(\"Done, do next step\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title (Optional) Connect Google Drive\n",
        "save_to_gdrive = False # @param {type:\"boolean\"}\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "if save_to_gdrive:\n",
        "    SAVE_DIRECTORY = \"/content/drive/MyDrive/guildedchatexporter/\"\n",
        "print(\"Done, do next step\")"
      ],
      "metadata": {
        "id": "6N-DxYLWPJG5",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Get Server status and Members for the chat\n",
        "memberlist = requests.get(SERVER_URL+\"/members\", headers=headers)\n",
        "memberlist.raise_for_status()\n",
        "#print(json.dumps(memberlist.json(), indent=4))\n",
        "print(\"Done, do next step\")\n",
        "\n"
      ],
      "metadata": {
        "id": "za3F-9r3YiMW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Get Messages\n",
        "def get_channel_messages(channel_id, after=None, limit=100):\n",
        "    params = {\n",
        "        'limit': limit,\n",
        "        'after': after\n",
        "    }\n",
        "    url = f'https://www.guilded.gg/api/v1/channels/{channel_id}/messages'\n",
        "    response = requests.get(url, headers=headers, params=params)\n",
        "    response.raise_for_status()\n",
        "    return response.json()\n",
        "\n",
        "def fetch_all_messages(channel_id):\n",
        "    all_messages = []\n",
        "    after = \"2015-01-01T00:00:00.000Z\"\n",
        "\n",
        "    while True:\n",
        "        data = get_channel_messages(channel_id, after=after, limit=100)\n",
        "        messages = data.get('messages', [])\n",
        "        all_messages.extend(messages)\n",
        "\n",
        "        if len(messages) < 100:\n",
        "            break\n",
        "\n",
        "        after = messages[-1]['createdAt']\n",
        "\n",
        "    return all_messages\n",
        "\n",
        "all_messages = fetch_all_messages(CHANNEL_ID)\n",
        "\n",
        "CHANNEL_URL = f'https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}'\n",
        "channelinfo = requests.get(CHANNEL_URL, headers=headers)\n",
        "channelinfo.raise_for_status()\n",
        "print(json.dumps(channelinfo.json(), indent=4))\n",
        "channelname = channelinfo.json()['channel']['name']\n",
        "channeltopic = channelinfo.json()['channel'].get('topic', '') # In case no topic is set\n",
        "# Figure out Usernames so we don't have gibberish\n",
        "members = memberlist.json()[\"members\"]\n",
        "\n",
        "\n",
        "print(f\"Found {len(all_messages)} messages.\")\n"
      ],
      "metadata": {
        "id": "YWZ59XqX94_u",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title (Optional) Print all messages and save raw results\n",
        "Print_JSON = False # @param {type:\"boolean\"}\n",
        "Save_JSON = True # @param {type:\"boolean\"}\n",
        "if Print_JSON:\n",
        "    print(json.dumps(all_messages, indent=4))\n",
        "if Save_JSON:\n",
        "    with open(f'{SAVE_DIRECTORY}{CHANNEL_ID}.json', 'w', encoding='utf-8') as file:\n",
        "        json.dump(all_messages, file, indent=4)\n",
        "        print(f\"JSON of all messages saved to {SAVE_DIRECTORY}{CHANNEL_ID}.json\")"
      ],
      "metadata": {
        "id": "FPAUteMhLGPO",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title (Optional) Save mildly-formatted .txt chat log\n",
        "\n",
        "with open(f'{SAVE_DIRECTORY}{CHANNEL_ID}.txt', 'w', encoding='utf-8') as file:\n",
        "    for message in all_messages:\n",
        "        created_by = message['createdBy']\n",
        "        created_at = message['createdAt']\n",
        "        content = message['content']\n",
        "        username = next((m['user']['name'] for m in members if m['user']['id'] == created_by), \"System Message\")\n",
        "        formatted_message = f\"{username} ({created_by}) ({created_at}): {content}\\n\"\n",
        "\n",
        "        file.write(formatted_message)\n",
        "\n",
        "print(\"Chat log saved\")"
      ],
      "metadata": {
        "id": "NQRjAHYpEBcJ",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title HTML File Setup\n",
        "# @markdown Has the base HTML and CSS information which you can edit. Just click RUN to set the info for the next step.\n",
        "# @markdown\n",
        "# @markdown This will have a flag in the future for if you don't want to download/show attachments, but why?\n",
        "## HTML Template\n",
        "\n",
        "CSS = \"\"\"\n",
        "@font-face {\n",
        "  font-family: \"Builder Sans\";\n",
        "  src: url(\"https://www.guilded.gg/fonts/BuilderSans-Regular.woff2\") format('woff2');\n",
        "}\n",
        "\n",
        "body {\n",
        "  font-family: 'Builder Sans', Tahoma, sans-serif;\n",
        "  color: white;\n",
        "  background-color: #373943;\n",
        "}\n",
        "\n",
        "#chat-log {\n",
        "  background-color: #373943;\n",
        "  padding: 16px;\n",
        "  display: flex;\n",
        "  flex-direction: column;\n",
        "  gap: 8px;\n",
        "}\n",
        "\n",
        ".message {\n",
        "  color: #ececee;\n",
        "  font-size: 15px;\n",
        "  line-height: 145%;\n",
        "  font-weight: normal;\n",
        "  font-family: 'Builder Sans', Tahoma, sans-serif;\n",
        "  margin-top: 0px;\n",
        "  min-height: 44px;\n",
        "  padding: 8px 12px;\n",
        "  display: flex;\n",
        "  align-content: flex-start;\n",
        "  gap: 8px;\n",
        "  transition: background-color 140ms ease;\n",
        "  position: relative;\n",
        "  contain: layout;\n",
        "  max-width: 80%;\n",
        "}\n",
        "\n",
        ".message:hover {\n",
        "  background-color: #31333c;\n",
        "}\n",
        "\n",
        ".created-at {\n",
        "  margin-right: 8px;\n",
        "  height: 11px;\n",
        "  color: #a3a3ac;\n",
        "  font-size: 13px;\n",
        "  line-height: 120%;\n",
        "  font-weight: normal;\n",
        "  white-space: nowrap;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".created-by {\n",
        "  font-size: 15px;\n",
        "  line-height: 145%;\n",
        "  font-weight: 700;\n",
        "  white-space: nowrap;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".pfp {\n",
        "  width: 40px;\n",
        "  height: 40px;\n",
        "  border-radius: 50%;\n",
        "  overflow: hidden;\n",
        "  vertical-align: middle;\n",
        "  flex-shrink: 0;\n",
        "  margin-right: 12px;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".content {\n",
        "  margin: 0;\n",
        "  word-break: break-all;\n",
        "}\n",
        "\n",
        ".content img,\n",
        ".content video {\n",
        "  max-height: 320px;\n",
        "  max-width: 360px;\n",
        "  object-fit: contain;\n",
        "}\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "HTML_TEMPLATE = \"\"\"\n",
        "<!DOCTYPE html>\n",
        "<html>\n",
        "<head>\n",
        "    <title>#{title}</title>\n",
        "    <style>\n",
        "        {CSS}\n",
        "    </style>\n",
        "</head>\n",
        "<body>\n",
        "    <div id=\"chat-log\">\n",
        "        <p>#{title}</p>\n",
        "        <p>{topic}</p>\n",
        "        <hr>\n",
        "        {messages}\n",
        "    </div>\n",
        "</body>\n",
        "</html>\n",
        "\"\"\"\n",
        "\n",
        "## Message Template\n",
        "MESSAGE_TEMPLATE = \"\"\"\n",
        "<div class=\"message\">\n",
        "    <img class=\"pfp\" src=\"{avatar_url}\">\n",
        "    <div>\n",
        "        <span class=\"created-by\">{created_by}</span>\n",
        "        <span class=\"created-at\">({created_at})</span>\n",
        "        <p class=\"content\">{content}</p>\n",
        "    </div>\n",
        "</div>\n",
        "\"\"\"\n",
        "\n",
        "print(\"HTML ready to print\")\n"
      ],
      "metadata": {
        "id": "tmKeILKmXJh8",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Parse and save results into HTML file!\n",
        "\n",
        "from urllib.parse import urlparse, unquote\n",
        "import random\n",
        "\n",
        "formatted_messages = []\n",
        "\n",
        "for message in all_messages:\n",
        "    created_by = message['createdBy']\n",
        "    created_at = message['createdAt']\n",
        "    content = message['content']\n",
        "\n",
        "    # Find inline attachments and replace them with relative links or embedded content\n",
        "    inline_attachments = re.findall(r'!\\[\\]\\((.*?)\\)', content)\n",
        "    for attachment in inline_attachments:\n",
        "        parsed_url = urlparse(attachment)\n",
        "        filename = os.path.basename(unquote(parsed_url.path))\n",
        "        relative_path = f'{CHANNEL_ID}/attachments/{filename}'\n",
        "\n",
        "        if filename.endswith('.webp'):\n",
        "            content = content.replace(f'![]({attachment})', f'<a href=\"{relative_path}\"><img src=\"{relative_path}\" alt=\"{filename}\"></a>')\n",
        "        elif filename.endswith('.mp4'):\n",
        "            content = content.replace(f'![]({attachment})', f'<video controls><source src=\"{relative_path}\" type=\"video/webm\">Your browser does not support the video tag.</video>')\n",
        "        else:\n",
        "            content = content.replace(f'![]({attachment})', f'<a href=\"{attachment}\" target=\"_blank\">{filename}(attachment)</a>')\n",
        "\n",
        "\n",
        "    username = next((m['user']['name'] for m in members if m['user']['id'] == created_by), \"System Message\")\n",
        "    avatar_url = next((m['user']['avatar'] for m in members if m['user']['id'] == created_by), \"https://www.guilded.gg/asset/DefaultUserAvatars/profile_5.png\")\n",
        "    avatar_filename = os.path.basename(unquote(urlparse(avatar_url).path))\n",
        "    avatar_url = f\"{CHANNEL_ID}/avatars/{avatar_filename}\"\n",
        "\n",
        "    formatted_message = MESSAGE_TEMPLATE.format(\n",
        "        avatar_url=avatar_url,\n",
        "        created_by=username,\n",
        "        created_at=created_at,\n",
        "        content=content\n",
        "    )\n",
        "    formatted_messages.append(formatted_message)\n",
        "\n",
        "\n",
        "html_content = HTML_TEMPLATE.format(CSS=CSS, title=channelname, topic=channeltopic, messages='\\n'.join(formatted_messages))\n",
        "\n",
        "with open(os.path.join(SAVE_DIRECTORY, f'{CHANNEL_ID}.html'), 'w', encoding='utf-8') as file:\n",
        "    file.write(html_content)\n",
        "\n",
        "print(f\"HTML file saved! Get it at {SAVE_DIRECTORY}{CHANNEL_ID}.html! Run the next step to get your attachments.\")"
      ],
      "metadata": {
        "id": "EG_eSrACZrs5",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Download all images/videos & member avatars\n",
        "# @markdown WARNING: Does not save file upload attachments that are not images or videos. Workaround might have to end up self-botting as Bot API simply can't interact with all other uploads.\n",
        "# @markdown\n",
        "# @markdown todo currently downloads all member pfps but we should not do that\n",
        "\n",
        "\n",
        "# Create the attachments folder if it doesn't exist\n",
        "attachments_dir = os.path.join(SAVE_DIRECTORY, CHANNEL_ID, 'attachments', )\n",
        "os.makedirs(attachments_dir, exist_ok=True)\n",
        "\n",
        "def download_attachments(messages, output_dir):\n",
        "    os.makedirs(output_dir, exist_ok=True)\n",
        "    url_pattern = r'!\\[(.*?)\\]\\((.*?)\\)'\n",
        "\n",
        "    for message in messages:\n",
        "        content = message['content']\n",
        "        matches = re.findall(url_pattern, content)\n",
        "\n",
        "        for match in matches:\n",
        "            url = match[1]\n",
        "            parsed_url = urlparse(url)\n",
        "            clean_filename = os.path.basename(unquote(parsed_url.path))\n",
        "            filename = os.path.join(output_dir, clean_filename)\n",
        "\n",
        "            try:\n",
        "                response = requests.get(url)\n",
        "                response.raise_for_status()\n",
        "\n",
        "                with open(filename, 'wb') as file:\n",
        "                    file.write(response.content)\n",
        "\n",
        "                print(f\"Downloaded attachment from {url} to {filename}\")\n",
        "            except requests.exceptions.RequestException as e:\n",
        "                print(f\"Error downloading attachment from {url}: {e}\")\n",
        "\n",
        "# Avatars\n",
        "avatars_dir = os.path.join(SAVE_DIRECTORY, CHANNEL_ID, 'avatars', )\n",
        "os.makedirs(avatars_dir, exist_ok=True)\n",
        "\n",
        "def download_avatars(members, output_dir):\n",
        "    os.makedirs(output_dir, exist_ok=True)\n",
        "\n",
        "    for member in members:\n",
        "        user = member['user']\n",
        "        if 'avatar' in user:\n",
        "            avatar_url = user['avatar']\n",
        "\n",
        "            if not avatar_url:\n",
        "                continue\n",
        "\n",
        "            parsed_url = urlparse(avatar_url)\n",
        "            clean_filename = os.path.basename(unquote(parsed_url.path))\n",
        "            filename = os.path.join(output_dir, clean_filename)\n",
        "\n",
        "            try:\n",
        "                response = requests.get(avatar_url)\n",
        "                response.raise_for_status()\n",
        "\n",
        "                with open(filename, 'wb') as file:\n",
        "                    file.write(response.content)\n",
        "\n",
        "                print(f\"Downloaded avatar from {avatar_url} to {filename}\")\n",
        "            except requests.exceptions.RequestException as e:\n",
        "                print(f\"Error downloading avatar from {avatar_url}: {e}\")\n",
        "\n",
        "all_messages = fetch_all_messages(CHANNEL_ID)\n",
        "output_dir = f'{SAVE_DIRECTORY}{CHANNEL_ID}'+'/attachments'\n",
        "download_attachments(all_messages, output_dir)\n",
        "download_avatars(members, avatars_dir)\n",
        "print(\"All done!\")"
      ],
      "metadata": {
        "id": "NFkiXR3ObAI0",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Pack all outputted files into ZIP and download! (And save to GDrive if enabled)\n",
        "\n",
        "%cd {SAVE_DIRECTORY}\n",
        "!zip -r {CHANNEL_ID}.zip {CHANNEL_ID}.json {CHANNEL_ID}.txt {CHANNEL_ID}.html {CHANNEL_ID}\n",
        "from google.colab import files\n",
        "files.download(f\"{CHANNEL_ID}.zip\")\n",
        "print(\"Hot and ready!\")\n",
        "\n",
        "try: # skip gdrive if user skipped it\n",
        "    if save_to_gdrive:\n",
        "        !cp {CHANNEL_ID}.zip /content/drive/MyDrive/guildedchatexporter/\n",
        "        print(f\"Copied {CHANNEL_ID}.zip to /content/drive/MyDrive/guildedchatexporter/\")\n",
        "except NameError:\n",
        "    pass\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "bkaYeiZVHiPR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title save forum channel entries to individual txt files\n",
        "# @markdown make sure you did \"Get Server status and Members for the chat\" first\n",
        "forum_url = f\"https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}/topics\"\n",
        "response = requests.get(forum_url, headers=headers, params={'limit': 100} )\n",
        "response.raise_for_status()\n",
        "#print(json.dumps(response.json(), indent=4))\n",
        "\n",
        "for forumTopic in response.json()['forumTopics']:\n",
        "  forumTopicId = forumTopic['id']\n",
        "  comments_url = f'https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}/topics/{forumTopicId}/comments'\n",
        "  comments_response = requests.get(comments_url, headers=headers, params={'limit': 100})\n",
        "  comments_response.raise_for_status()\n",
        "  #print(json.dumps(comments_response.json(), indent=4))\n",
        "\n",
        "\n",
        "members = memberlist.json()[\"members\"]\n",
        "for forumTopic in response.json()['forumTopics']:\n",
        "  forumTopicId = forumTopic['id']\n",
        "  forumtitle = forumTopic['title']\n",
        "  forumcreated_by = forumTopic['createdBy']\n",
        "  forumcreated_at = forumTopic['createdAt']\n",
        "\n",
        "  firsturl = f'https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}/topics/{forumTopicId}'\n",
        "  firstresp = requests.get(firsturl, headers=headers)\n",
        "\n",
        "\n",
        "  comments_url = f'https://www.guilded.gg/api/v1/channels/{CHANNEL_ID}/topics/{forumTopicId}/comments'\n",
        "  comments_response = requests.get(comments_url, headers=headers, params={'limit': 100})\n",
        "\n",
        "  #print(json.dumps(comments_response.json(), indent=4))\n",
        "  txttitle = re.sub(r'[<>:\"/\\\\|?*]', '_', forumtitle)\n",
        "  os.makedirs(os.path.dirname(f'{SAVE_DIRECTORY}forum/{CHANNEL_ID}/{txttitle}.txt'), exist_ok=True)\n",
        "  with open(f'{SAVE_DIRECTORY}forum/{CHANNEL_ID}/{txttitle}.txt', 'w', encoding='utf-8') as file:\n",
        "    #write the title, then newline, username, createdby, createdat\n",
        "    file.write(f\"{forumtitle}\\n\\n\")\n",
        "    #get the right username for the first poster\n",
        "    username = next((m['user']['name'] for m in members if m['user']['id'] == forumcreated_by), \"[deleted user]\")\n",
        "    file.write(f\"{username} ({forumcreated_by}) ({forumcreated_at}): {firstresp.json()['forumTopic']['content']}\\n\")\n",
        "\n",
        "\n",
        "    for message in comments_response.json()['forumTopicComments']:\n",
        "        created_by = message['createdBy']\n",
        "        created_at = message['createdAt']\n",
        "        content = message['content']\n",
        "        username = next((m['user']['name'] for m in members if m['user']['id'] == created_by), \"[deleted user]\")\n",
        "        formatted_message = f\"{username} ({created_by}) ({created_at}): {content}\\n\"\n",
        "\n",
        "        file.write(formatted_message)\n",
        "        # notify when end of comments_response reached\n",
        "\n",
        "print(f\"done saving to /guildedchatexporter/forum/CHANNEL_ID\")\n",
        "\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "EhPlEWZS7j1e"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}