guildedchatexporter2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Fatih120/GuildedChatExporter/blob/main/guildedchatexporter2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ko9P7YC-pbci"
      },
      "source": [
        "# GuildedChatExporter v2\n",
        "\n",
        "Export your stuff before it's gone.\n",
        "Full Transparency : 80% done with AI.\n",
        "\n",
        "Todo: cleanup, less crap code, HTML forums and output other at least in txt (unlikely)\n",
        "\n",
        "Need help?\n",
        "https://github.com/Fatih120/GuildedChatExporter/tree/main\n",
        "\n",
        "\n",
        "[Discord Server :(](https://discord.com/invite/Cy27FNfQtc)\n",
        "\n",
        "IF THERE ARE ANY ISSUES PLEASE DON'T HESITATE TO CONTACT.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "smm3F6pT1WpC",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "# @title Basic Setup\n",
        "\n",
        "# @markdown 1.   Log on to Guilded on your web browser, not client, and focus on that tab.\n",
        "\n",
        "# @markdown 2.   If based on **Firefox**, press `SHIFT + F9`.\n",
        "# @markdown\n",
        "# @markdown If on ***Chrome***, press `F12` and find the \"Application\" tab.\n",
        "# @markdown\n",
        "# @markdown We're here to grab your login, so click the Cookies entry on the sidebar, and then the guilded.gg url.\n",
        "# @markdown\n",
        "# @markdown Find the `hmac_signed_session` key at the bottom. Copy the value, it is very long. Paste it here.\n",
        "\n",
        "#!apt-get install -y p7zip-full\n",
        "import requests\n",
        "import json\n",
        "import re\n",
        "import os\n",
        "import shutil\n",
        "import urllib.parse\n",
        "import ipywidgets as widgets\n",
        "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
        "from google.colab import files\n",
        "import urllib.parse\n",
        "from urllib.parse import urlparse\n",
        "!pip install emoji\n",
        "import emoji\n",
        "\n",
        "USER_TOKEN = \"XXXX.YYYY.ZZZZ\" # @param {type:\"string\"}\n",
        "# @markdown Do NOT share this value with anyone as they will be able to access your account without a password.\n",
        "\n",
        "SAVE_DIRECTORY = \"/guildedchatexporter/\" # @param {type:\"string\"}\n",
        "if not os.path.exists(SAVE_DIRECTORY):\n",
        "  os.makedirs(SAVE_DIRECTORY)\n",
        "\n",
        "cookies = {\n",
        "    \"authenticated\": \"true\",\n",
        "    \"hmac_signed_session\": USER_TOKEN,\n",
        "}\n",
        "\n",
        "\n",
        "# @markdown Google Colab gives you over 100GB of temporary space to work with, which should be way more than enough to store and then download your stuff.\n",
        "# @markdown\n",
        "# @markdown  You can also connect your Google Drive instead and copy the stuff to there at the end, though if you just want to download your files immediately without using Google Drive, skip the optional step.\n",
        "# --------------------------------------------------- #\n",
        "\n",
        "guilded = \"https://www.guilded.gg/api\"\n",
        "def fetch(endpoint, params=None):\n",
        "    url = f\"{guilded}/{endpoint}\"\n",
        "    response = requests.get(url, params=params, cookies=cookies)\n",
        "    if response.status_code == 200:\n",
        "      return unshid_cdn(response.json())\n",
        "    else:\n",
        "        raise Exception(f\"Error fetching data: {response.status_code} {response.text}\")\n",
        "        print(\"Did you correctly input your token?\")\n",
        "\n",
        "def unshid_cdn(data):\n",
        "  def fix_url(url):\n",
        "    match = re.match(r\"^https://s3-us-west-2\\.amazonaws\\.com/www\\.guilded\\.gg/(.*)$\", url)\n",
        "    if match:\n",
        "      return f\"https://cdn.gilcdn.com/{match.group(1)}\"\n",
        "    else:\n",
        "      return url\n",
        "\n",
        "  def fix_dict(d):\n",
        "    for key, value in d.items():\n",
        "      if isinstance(value, dict):\n",
        "        fix_dict(value)\n",
        "      elif isinstance(value, list):\n",
        "        for item in value:\n",
        "          if isinstance(item, dict):\n",
        "            fix_dict(item)\n",
        "          elif isinstance(item, str):\n",
        "            d[key] = fix_url(item)\n",
        "      elif isinstance(value, str):\n",
        "        d[key] = fix_url(value)\n",
        "\n",
        "  fix_dict(data)\n",
        "  return data\n",
        "\n",
        "def fetch_user(user_id):\n",
        "    data = fetch(f\"users/{user_id}\")\n",
        "    return data[\"user\"]\n",
        "\n",
        "def fetch_channel(channel_id):\n",
        "    data = fetch(\"content/route/metadata\", params={\"route\": f\"//channels/{channel_id}/chat\"})\n",
        "    return data[\"metadata\"][\"channel\"]\n",
        "\n",
        "def fetch_servers():\n",
        "    data = fetch(\"me\", params={\"isLogin\": \"false\", \"v2\": \"true\"})\n",
        "    return data['teams']\n",
        "\n",
        "def fetch_members():\n",
        "    data = fetch(f\"teams/{SERVER_ID}/members\")\n",
        "    return data\n",
        "\n",
        "def get_groups():\n",
        "    return fetch(f\"teams/{SERVER_ID}/groups\")\n",
        "\n",
        "def get_groups():\n",
        "    return fetch(f\"teams/{SERVER_ID}/groups\")\n",
        "\n",
        "def get_channels():\n",
        "    return fetch(f\"teams/{SERVER_ID}/channels\")\n",
        "\n",
        "def fetch_dms():\n",
        "    return fetch(f\"users/{USER_ID}/channels\")\n",
        "\n",
        "def fetch_myself():\n",
        "    return fetch(\"me\", params={\"isLogin\": \"false\", \"v2\": \"true\"})\n",
        "\n",
        "def fetch_info():\n",
        "    return fetch(f\"teams/{SERVER_ID}/info\")\n",
        "\n",
        "def sanitize_filename(filename):\n",
        "    filename = re.sub(r'[<>:\"/\\\\|?*]', '_', filename)\n",
        "    filename = filename.rstrip(' _')\n",
        "    if not filename:\n",
        "        filename = 'untitled'\n",
        "    return filename[:250]\n",
        "\n",
        "from IPython.display import clear_output\n",
        "clear_output()\n",
        "\n",
        "print(\"Please select the server you wish to archive in the dropdown.\")\n",
        "server_dict = {f\"{server['name']} ({server['id']})\": server[\"id\"] for server in fetch_servers()}\n",
        "server_dict[''] = None\n",
        "server_dropdown = widgets.Dropdown(options=list(server_dict.keys()), value='', description='Servers:',)\n",
        "display(server_dropdown)\n",
        "\n",
        "def on_server_select(change):\n",
        "    global SERVER_ID\n",
        "    global SERVER_NAME\n",
        "    selected_server_name = change[\"new\"]\n",
        "    SERVER_ID = server_dict[selected_server_name]\n",
        "    SERVER_NAME = selected_server_name\n",
        "    print(f\"Selected server: {selected_server_name}\")\n",
        "    print(\"You can go to the next step, or you can go straight for DMs\")\n",
        "\n",
        "server_dropdown.observe(on_server_select, names=\"value\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "nbzod6w0ypF0"
      },
      "outputs": [],
      "source": [
        "# @title Save Basic Data about Server and Users\n",
        "if 'SERVER_ID' not in globals():\n",
        "    print(\"You forgot to choose a server. Go back and select one from the dropdown.\")\n",
        "else:\n",
        "    server_dir = os.path.join(SAVE_DIRECTORY, SERVER_NAME)\n",
        "    os.makedirs(server_dir, exist_ok=True)\n",
        "\n",
        "\n",
        "verbose = True # @param {type:\"boolean\"}\n",
        "get_audits = False # @param {type:\"boolean\"}\n",
        "# @markdown You can try to get audit logs for the server, but it'll fail if you're not authorized to see them.\n",
        "\n",
        "# @markdown **If you know the names of certain [deleted user]s on the server and don't want their messages to appear as such, note their IDs and edit the members.json found in the server save directory. All the deleted members are stored in there, so just replace their name with the one you wish for.**\n",
        "\n",
        "def download_file(url, filepath):\n",
        "    !wget -q -nc -O \"$filepath\" \"$url\"\n",
        "mc = 0\n",
        "members_data = fetch_members()\n",
        "members = members_data.get('members', [])\n",
        "\n",
        "overview = fetch(f\"teams/{SERVER_ID}/overview\")\n",
        "with open(os.path.join(server_dir, 'overview.json'), 'w') as f:\n",
        "    f.write(json.dumps(overview, indent=4))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}{SERVER_NAME}/overview.json\")\n",
        "\n",
        "\n",
        "all_audit_logs = []\n",
        "before_date = None\n",
        "if get_audits == True:\n",
        "    print(\"Getting audit logs...\")\n",
        "    while True:\n",
        "        params = {\"maxItems\": 50}\n",
        "        if before_date:\n",
        "            params[\"beforeDate\"] = before_date\n",
        "        response = requests.post(f\"https://www.guilded.gg/api/teams/{SERVER_ID}/auditlogs\", json=params, cookies=cookies)\n",
        "        response.raise_for_status()\n",
        "        audit_logs = response.json()\n",
        "        new_logs = audit_logs[\"logs\"]\n",
        "        all_audit_logs.extend(new_logs)\n",
        "        if len(new_logs) < 50:\n",
        "            break\n",
        "        !sleep 1\n",
        "        before_date = new_logs[-1][\"createdAt\"]\n",
        "    with open(os.path.join(server_dir, 'auditlogs.json'), 'w') as f:\n",
        "        json.dump(all_audit_logs, f, indent=4)\n",
        "    print(f\"Saved {len(all_audit_logs)} audit logs to {os.path.join(server_dir, 'auditlogs.json')}\")\n",
        "\n",
        "\n",
        "\n",
        "for member in members:\n",
        "    member_id = member[\"id\"]\n",
        "    member_name = member[\"name\"]\n",
        "    member_dir = os.path.join(server_dir, \"members\", member_name)\n",
        "    os.makedirs(member_dir, exist_ok=True)\n",
        "    user_data = fetch_user(member_id)\n",
        "\n",
        "    with open(os.path.join(member_dir, f\"{member_id}.json\"), 'w') as f:\n",
        "        f.write(json.dumps(user_data, indent=4))\n",
        "    json_string = json.dumps(user_data)\n",
        "\n",
        "    cdn_links = re.findall(r\"https://cdn\\.gilcdn\\.com/[^\\\"]+\", json_string)\n",
        "    with ThreadPoolExecutor(max_workers=4) as executor:\n",
        "        futures = [executor.submit(download_file, url, os.path.join(member_dir, url.split(\"/\")[-1].split(\"?\")[0])) for url in cdn_links]\n",
        "\n",
        "    mc += 1\n",
        "    if verbose:\n",
        "      print(f\"{str(mc)}/\" + str(len(members)) + f\" {member_name} ({member_id})\")\n",
        "\n",
        "\n",
        "\n",
        "with open(os.path.join(server_dir, 'members.json'), 'w') as f:\n",
        "    f.write(json.dumps(fetch_members(), indent=4))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}{SERVER_NAME}/members.json\")\n",
        "with open(os.path.join(server_dir, 'info.json'), 'w') as f:\n",
        "    infourls = []\n",
        "    info = fetch_info()\n",
        "    f.write(json.dumps(info, indent=4))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}{SERVER_NAME}/info.json\")\n",
        "    if 'profilePicture' in info['team']:\n",
        "        infourls.append(info['team']['profilePicture'])\n",
        "    if 'teamDashImage' in info['team']:\n",
        "        infourls.append(info['team']['teamDashImage'])\n",
        "    if 'homeBannerImageLg' not in info['team']:\n",
        "        if 'homeBannerImageMd' in info['team']:\n",
        "            infourls.append(info['team']['homeBannerImageMd'])\n",
        "        if 'homeBannerImageSm' in info['team']:\n",
        "            infourls.append(info['team']['homeBannerImageSm'])\n",
        "    else:\n",
        "        infourls.append(info['team']['homeBannerImageLg'])\n",
        "    for url in infourls:\n",
        "        if verbose:\n",
        "            print(f\"checking: {type(url)}, {url}\")\n",
        "        if isinstance(url, bytes):\n",
        "            print(f\"this is apparently a BYTE URL: {url}\")\n",
        "            continue\n",
        "\n",
        "        try:\n",
        "            parsed_url = urlparse(url)\n",
        "            basename = os.path.basename(parsed_url.path)\n",
        "            full_path = os.path.join(server_dir, basename)\n",
        "            if verbose:\n",
        "                print(f\"Basename: {type(basename)}, {basename}\")\n",
        "                print(f\"Full path: {type(full_path)}, {full_path}\")\n",
        "            download_file(url, full_path)\n",
        "            print(f\"Successful: {url}\")\n",
        "        except Exception as e:\n",
        "            print(f\"Error with URL {url}: {str(e)}\")\n",
        "\n",
        "with open(os.path.join(server_dir, 'groups.json'), 'w') as f:\n",
        "    f.write(json.dumps(get_groups(), indent=4))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}{SERVER_NAME}/groups.json\")\n",
        "with open(os.path.join(server_dir, 'channels.json'), 'w') as f:\n",
        "    f.write(json.dumps(get_channels(), indent=4))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}{SERVER_NAME}/channels.json\")\n",
        "\n",
        "print(\"Done getting some fundamental stuff, go to the next step\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "yXPb_jtIKbMw"
      },
      "outputs": [],
      "source": [
        "# @title Save All Emoji\n",
        "from google.colab import files\n",
        "emotes_data = fetch(f\"teams/{SERVER_ID}/customReactions\")\n",
        "\n",
        "verbose = True # @param {type:\"boolean\"}\n",
        "emotes_dir = os.path.join(server_dir, \"emotes\")\n",
        "download_immediately = False # @param {type:\"boolean\"}\n",
        "\n",
        "def fetch_emotes():\n",
        "    os.makedirs(emotes_dir, exist_ok=True)\n",
        "\n",
        "    with open(os.path.join(server_dir, 'emotes.json'), 'w') as f:\n",
        "        f.write(json.dumps(emotes_data, separators=(',', ':')))\n",
        "    print(\"Saved emotes.json\")\n",
        "\n",
        "    ec = 0\n",
        "    with ThreadPoolExecutor(max_workers=4) as executor:\n",
        "        futures = []\n",
        "        downloaded = set()  # need to check if we already got it cuz dupes, waste less time\n",
        "        for reaction in emotes_data.get('reactions', []):\n",
        "            for image_type in [\"png\", \"webp\", \"apng\"]:\n",
        "                url = reaction.get(image_type)\n",
        "                if url and url not in downloaded:\n",
        "                    filename = url.split(\"/\")[-1].split(\"?\")[0]\n",
        "                    file_extension = filename.split(\".\")[-1]\n",
        "                    filepath = os.path.join(emotes_dir, f\"{reaction['id']}-{reaction['name']}.{file_extension}\")\n",
        "\n",
        "                    ## Check if the file already exists\n",
        "                    #if os.path.exists(filepath):\n",
        "                    #    # If it exists, save it using the ID instead\n",
        "                    #    filepath = os.path.join(emotes_dir, f\"{reaction['id']}.{file_extension}\")\n",
        "                    #    print(f\"Conflicting emote: {reaction['name']} (ID: {reaction['id']})\")\n",
        "                    #    print(f\"Saving the emote as an ID, this is the sacrifice to make if we want to keep the real name.\")\n",
        "\n",
        "                    futures.append(executor.submit(download_emote, url, filepath))\n",
        "                    downloaded.add(url)\n",
        "                    ec += 1\n",
        "                    if verbose:\n",
        "                      print(f\"{ec}/{str(len(emotes_data.get('reactions', [])))} \" + reaction[\"name\"])\n",
        "\n",
        "        for future in as_completed(futures):\n",
        "            future.result()\n",
        "\n",
        "        print(\"All done with server emotes!\")\n",
        "\n",
        "def download_emote(url, filepath):\n",
        "    !wget -q -nc -O \"$filepath\" \"$url\"\n",
        "\n",
        "fetch_emotes()\n",
        "\n",
        "\n",
        "# This is so tragic omg.\n",
        "\n",
        "role_icons_dir = os.path.join(emotes_dir, \"roleIcons\")\n",
        "os.makedirs(role_icons_dir, exist_ok=True)\n",
        "with open(os.path.join(server_dir, \"info.json\"), \"r\") as f:\n",
        "    info_data = json.load(f)\n",
        "os.makedirs(role_icons_dir, exist_ok=True)\n",
        "print(f\"Getting external emoji\")\n",
        "\n",
        "for role_data in info_data.get(\"team\", {}).get(\"rolesById\", {}).values():\n",
        "    icon_url = role_data.get(\"iconUrl\")\n",
        "    if icon_url:\n",
        "        if icon_url.startswith(\"http\"): # Check if it looks like a valid URL\n",
        "            filename = os.path.basename(urlparse(icon_url).path)\n",
        "            daname = os.path.join(f\"{role_data.get('id')}-\", filename)\n",
        "            download_path = os.path.join(role_icons_dir, filename)\n",
        "            with open(download_path, \"wb\") as f:\n",
        "                f.write(requests.get(icon_url).content)\n",
        "\n",
        "if download_immediately:\n",
        "    emojizip = os.path.join(SAVE_DIRECTORY, f\"{SERVER_NAME}_emoji.zip\")\n",
        "    shutil.make_archive(emojizip[:-4], 'zip', emotes_dir)\n",
        "    files.download(emojizip)\n",
        "\n",
        "print(\"Done grabbing emoji, go to next step\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "dxgFWcL0SBzq"
      },
      "outputs": [],
      "source": [
        "# @title HTML File Setup\n",
        "# @markdown Has the base HTML and CSS information which you can edit. Just click RUN to set the info for the next step.\n",
        "# @markdown\n",
        "## HTML Template\n",
        "\n",
        "CSS = \"\"\"\n",
        "@font-face {\n",
        "  font-family: \"Builder Sans\";\n",
        "  src: url(\"https://www.guilded.gg/fonts/BuilderSans-Regular.woff2\") format('woff2');\n",
        "}\n",
        "\n",
        "body {\n",
        "  font-family: 'Builder Sans', Tahoma, sans-serif;\n",
        "  color: white;\n",
        "  background-color: #373943;\n",
        "}\n",
        "\n",
        "#chat-log {\n",
        "  background-color: #373943;\n",
        "  padding: 16px;\n",
        "  display: flex;\n",
        "  flex-direction: column;\n",
        "  gap: 8px;\n",
        "}\n",
        "\n",
        ".message {\n",
        "  color: #ececee;\n",
        "  font-size: 15px;\n",
        "  line-height: 145%;\n",
        "  font-weight: normal;\n",
        "  font-family: 'Builder Sans', Tahoma, sans-serif;\n",
        "  margin-top: 0px;\n",
        "  min-height: 44px;\n",
        "  padding: 8px 12px;\n",
        "  display: flex;\n",
        "  align-content: flex-start;\n",
        "  gap: 8px;\n",
        "  transition: background-color 140ms ease;\n",
        "  position: relative;\n",
        "  contain: layout;\n",
        "  max-width: 80%;\n",
        "}\n",
        "\n",
        ".message:hover {\n",
        "  background-color: #31333c;\n",
        "}\n",
        "\n",
        ".created-at {\n",
        "  margin-right: 8px;\n",
        "  height: 11px;\n",
        "  color: #a3a3ac;\n",
        "  font-size: 13px;\n",
        "  line-height: 120%;\n",
        "  font-weight: normal;\n",
        "  white-space: nowrap;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".created-by {\n",
        "  font-size: 15px;\n",
        "  line-height: 145%;\n",
        "  font-weight: 700;\n",
        "  white-space: nowrap;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".pfp {\n",
        "  width: 40px;\n",
        "  height: 40px;\n",
        "  border-radius: 50%;\n",
        "  overflow: hidden;\n",
        "  vertical-align: middle;\n",
        "  flex-shrink: 0;\n",
        "  margin-right: 12px;\n",
        "  pointer-events: auto;\n",
        "}\n",
        "\n",
        ".emote {\n",
        "  height: 1.3em;\n",
        "  vertical-align: middle;\n",
        "}\n",
        "\n",
        ".content {\n",
        "  margin: 0;\n",
        "  word-break: break-all;\n",
        "}\n",
        "\n",
        ".content img,\n",
        ".content video {\n",
        "  max-height: 320px;\n",
        "  max-width: 360px;\n",
        "  object-fit: contain;\n",
        "}\n",
        "\n",
        ".file-upload-container {\n",
        "  display: flex;\n",
        "  align-items: center;\n",
        "  background-color: #32343d;\n",
        "  padding: 10px;\n",
        "  border: 1px #a3a3ac solid;\n",
        "  border-radius: 5px;\n",
        "\tcolor: #fef0b9;\n",
        "  margin-top: 12px;\n",
        "  margin-bottom: -40px;\n",
        "}\n",
        "\n",
        ".file-icon {\n",
        "  margin-right: 10px;\n",
        "  background-size: contain;\n",
        "}\n",
        "\n",
        ".file-info {\n",
        "    flex-grow: 1;\n",
        "}\n",
        "\n",
        ".file-name {\n",
        "    font-weight: bold;\n",
        "    text-decoration: none;\n",
        "    color: #fef0b9;\n",
        "}\n",
        "\n",
        ".file-size {\n",
        "    color: #666;\n",
        "    font-size: 0.9em;\n",
        "}\n",
        "\n",
        ".download-icon {\n",
        "    margin-left: 10px;\n",
        "    color: #666;\n",
        "}\n",
        "\n",
        ".icon {\n",
        "    width: 22px;\n",
        "    height: 22px;\n",
        "    background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAAWCAMAAAD6gTxzAAAAk1BMVEUAAADt7e3u7u7w8PDy8vLo6PPp6fDt7e3p6e/r6/Dt7e3t7e3s7Ozt7e3t7e3s7O/s7O/t7e3t7e3s7O/r6+7r6+7t7e/t7e3r6+3r6+3s7O7s7O/s7O7s7O7t7e7s7O7s7O/t7e7s7O7s7O7s7O7s7O7t7e7r6+7s7O7s7O/s7O7s7O7s7O7s7O7s7O7s7O7////r1//TAAAAL3RSTlMADg8RExYiKi8zODlERUhQUVVWX2Znb3FzdHh8h5KZoarBx8jMzs/Q19nq7vD6/rP6L04AAAABYktHRDCu3C3kAAAAiklEQVQY04XR2RKCMAyF4eCCK+KOoOCKu+T9305KT0MHmOG/6nyT5qKlyYuLfqlP6H5ZqzbHjCPQd4tDwsaElnyACc25E/K+SqTNoqHrJuquRTq/pP40bxZn55LQ6V2j3acgb4E8okDTA5s5FXK6yBGyAw1Gqp5NeteqZaphV4Xw0NL1RuY7TM/xH0oeGJMDGiugAAAAAElFTkSuQmCC');\n",
        "    background-repeat: no-repeat;\n",
        "}\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "HTML_TEMPLATE = \"\"\"\n",
        "<!DOCTYPE html>\n",
        "<html>\n",
        "<head>\n",
        "    <title>#{title}</title>\n",
        "    <style>\n",
        "        {CSS}\n",
        "    </style>\n",
        "</head>\n",
        "<body>\n",
        "    <div id=\"chat-log\">\n",
        "        <p>#{title}</p>\n",
        "        <p>{topic}</p>\n",
        "        <hr>\n",
        "        {messages}\n",
        "    </div>\n",
        "</body>\n",
        "</html>\n",
        "\"\"\"\n",
        "\n",
        "## Message Template\n",
        "MESSAGE_TEMPLATE = \"\"\"\n",
        "<div class=\"message\">\n",
        "    <img class=\"pfp\" src=\"{avatar_url}\">\n",
        "    <div>\n",
        "        <span class=\"created-by\">{created_by}</span>\n",
        "        <span class=\"created-at\">({created_at})</span>\n",
        "        <p class=\"content\">{content}</p>\n",
        "    </div>\n",
        "</div>\n",
        "\"\"\"\n",
        "\n",
        "print(\"HTML ready to print\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wdTrTab-Is4e"
      },
      "source": [
        "# Servers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q-PtNAY1hihd",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "# @title Download all Media and Save HTMLs for every channel\n",
        "messages_per_page = 10000 # @param {type:\"integer\"}\n",
        "# @markdown This is the breadwinner - we will finally save and print everything, with text/stream/voice channels outputting as HTML files. It's huge and lengthy, so if you have an especially big server on top of that, you will probably want to leave and get a coffee while this runs.\n",
        "\n",
        "# @markdown This will also split HTML files by the amount of messages, because you really don't wanna open a file over 10Mb large in your browser. 10000 should be a nice default and still might be too laggy depending on your channel, so increase it or decrease it as you need. Nothing will happen if your channel has less messages than this.\n",
        "\n",
        "# @markdown Every other channel type will be saved only to JSONs. It isn't worth the effort to convert those. Since you will have this raw data, though, if anyone is up for it, this data can be processed later, so as long as you have the JSONs, you pretty much have everything. All media downloads will be saved as well. (I believe.)\n",
        "from urllib.parse import unquote, urlparse\n",
        "verbose_mescount = True # @param {type:\"boolean\"}\n",
        "\n",
        "doChats          = True # @param {type:\"boolean\"}\n",
        "doEvents         = True # @param {type:\"boolean\"}\n",
        "doSchedules      = True # @param {type:\"boolean\"}\n",
        "doMedia          = True # @param {type:\"boolean\"}\n",
        "doDocs           = True # @param {type:\"boolean\"}\n",
        "doAnnouncements  = True # @param {type:\"boolean\"}\n",
        "doLists          = True # @param {type:\"boolean\"}\n",
        "doForums         = True # @param {type:\"boolean\"}\n",
        "\n",
        "# Load emotes data\n",
        "with open(os.path.join(server_dir, \"emotes.json\"), \"r\") as f:\n",
        "    emotes_data = json.load(f)\n",
        "    emotes_dict = {str(emote['id']): emote for emote in emotes_data.get('reactions', [])}\n",
        "# Load server info for roles\n",
        "with open(os.path.join(server_dir, \"info.json\"), \"r\") as f:\n",
        "    server_info = json.load(f)\n",
        "    roles_iterable = server_info['team']['rolesById'].values() if isinstance(server_info['team']['rolesById'], dict) else server_info['team']['rolesById']\n",
        "    roles_data = {str(role['id']): role for role in roles_iterable}\n",
        "# Load members data\n",
        "with open(os.path.join(server_dir, \"members.json\"), \"r\") as f:\n",
        "    members_json = json.load(f)\n",
        "    members_dict = {member['id']: member for member in members_json['members']}\n",
        "# Load channels data\n",
        "with open(os.path.join(server_dir, \"channels.json\"), \"r\") as f:\n",
        "    channels_data = json.load(f)\n",
        "with open(os.path.join(server_dir, \"groups.json\"), \"r\") as f:\n",
        "    groups_data = json.load(f)\n",
        "\n",
        "def download_file(url, filename):\n",
        "    response = requests.get(url)\n",
        "    if response.status_code == 200:\n",
        "        os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
        "        with open(filename, \"wb\") as f:\n",
        "            f.write(response.content)\n",
        "    else:\n",
        "        print(f\"Error fetching data from '{url}': Status code {response.status_code}\")\n",
        "        return None\n",
        "\n",
        "def get_colored_name(member_id, name):\n",
        "    def get_member_color(member):\n",
        "        if member and 'roleIds' in member:\n",
        "            for role_id in member['roleIds']:\n",
        "                role = roles_data.get(str(role_id))\n",
        "                if role and role['color'] != 'transparent':\n",
        "                    return role['color']\n",
        "        return None\n",
        "\n",
        "    member = next((m for m in members_data['members'] if m['id'] == member_id), None)\n",
        "    color = get_member_color(member)\n",
        "\n",
        "    if color:\n",
        "        return f'<span style=\"color: {color};\">@{name}</span>'\n",
        "    return f'@{name}'\n",
        "\n",
        "def shit(url):\n",
        "    match = re.match(r\"^https://s3-us-west-2\\.amazonaws\\.com/www\\.guilded\\.gg/(.+)$\", url)\n",
        "    if match:\n",
        "        return f\"https://cdn.gilcdn.com/{match.group(1)}\"\n",
        "    return url\n",
        "\n",
        "def dontcare(document):\n",
        "    if isinstance(document, dict):\n",
        "        for key, value in document.items():\n",
        "            if key == \"src\" and isinstance(value, str):\n",
        "                document[key] = shit(value)\n",
        "            else:\n",
        "                dontcare(value)\n",
        "    elif isinstance(document, list):\n",
        "        for item in document:\n",
        "            dontcare(item)\n",
        "\n",
        "def lmao(url, save_path, cookies):\n",
        "    try:\n",
        "        media_response = requests.get(url, cookies=cookies)\n",
        "        media_response.raise_for_status()\n",
        "        with open(save_path, \"wb\") as f:\n",
        "            f.write(media_response.content)\n",
        "        if verbose_mescount:\n",
        "            print(f\"Downloaded {os.path.basename(save_path)}\")\n",
        "    except Exception as e:\n",
        "        print(f\"Error downloading {os.path.basename(save_path)}: {e}\")\n",
        "\n",
        "def generate_html(messages, channel_data, members_data, server_dir, channel_dir):\n",
        "    formatted_messages = []\n",
        "    doge = {}\n",
        "\n",
        "    with ThreadPoolExecutor() as executor:\n",
        "        futures = []\n",
        "\n",
        "        for message in messages:\n",
        "            created_by_id = message['createdBy']\n",
        "            created_at = message.get('createdAt') or message.get('created_at')\n",
        "\n",
        "            # Determine if it's a system message\n",
        "            is_system_message = message.get('type') == 'system' or 'systemMessage' in str(message.get('content', {}))\n",
        "\n",
        "            if is_system_message:\n",
        "                member = next((m for m in members_data['members'] if m['id'] == created_by_id), None)\n",
        "                created_by = member[\"name\"] if member else \"[deleted user]\"\n",
        "                avatar_url = \"https://www.guilded.gg/asset/DefaultUserAvatars/profile_5.png\"\n",
        "\n",
        "                # Handle system message content\n",
        "                content = \"\"\n",
        "                document = message.get('content', {}).get('document', {})\n",
        "                for node in document.get('nodes', []):\n",
        "                    if node.get('type') == 'systemMessage':\n",
        "                        data = node.get('data', {})\n",
        "                        message_type = data.get('type')\n",
        "                        if message_type == 'team-channel-created':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) created this chat channel.\"\n",
        "                        elif message_type == 'channel-renamed':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) renamed this channel from '{data.get('oldName')}' to '{data.get('newName')}'.\"\n",
        "                        elif message_type == 'streaming-screenshare-started':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) started to share their screen.\"\n",
        "                        elif message_type == 'voice-call-started':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) started a voice call.\"\n",
        "                        elif message_type == 'webhookMessage':\n",
        "                            embeds = node.get('data', {}).get('embeds', [])\n",
        "                            content = json.dumps(data, indent=2)\n",
        "                        else:\n",
        "                            content = f\"System action performed: {message_type}\"\n",
        "            else:\n",
        "                # Find member data based on createdBy ID\n",
        "                member = next((m for m in members_data['members'] if m['id'] == created_by_id), None)\n",
        "                created_by = member[\"name\"] if member else \"[deleted user]\"\n",
        "\n",
        "                # Get the base role\n",
        "                base_role = next((role for role in roles_data.values() if role.get('isBase')), None)\n",
        "                base_color = base_role[\"color\"] if base_role else None\n",
        "\n",
        "                # Get the highest priority role for the member\n",
        "                highest_priority_role = None\n",
        "                if member and 'roleIds' in member:\n",
        "                    member_roles = [roles_data.get(str(role_id)) for role_id in member[\"roleIds\"] if str(role_id) in roles_data]\n",
        "                    member_roles.sort(key=lambda x: x[\"priority\"], reverse=True)\n",
        "                    if member_roles:\n",
        "                        highest_priority_role = member_roles[0]\n",
        "\n",
        "                # Color the created_by name based on the highest priority role or base role\n",
        "                if highest_priority_role and highest_priority_role[\"color\"] != \"transparent\":\n",
        "                    color = highest_priority_role[\"color\"]\n",
        "                elif base_role:\n",
        "                    color = base_role[\"color\"]\n",
        "                else:\n",
        "                    color = None\n",
        "\n",
        "                if color:\n",
        "                    created_by = f'<span style=\"color: {color};\">{created_by}</span>'\n",
        "                else:\n",
        "                    created_by = created_by\n",
        "\n",
        "\n",
        "                # Handle avatar URL\n",
        "                if created_by_id in members_dict and members_dict[created_by_id].get('profilePicture'):\n",
        "                    avatar_url = os.path.join(\"..\", \"..\", \"members\", members_dict[created_by_id].get('name'), os.path.basename(members_dict[created_by_id].get('profilePicture')).split('?')[0])\n",
        "                else:\n",
        "                    if created_by_id not in doge:\n",
        "                        avatar_number = (len(doge) % 5) + 1\n",
        "                        doge[created_by_id] = avatar_number\n",
        "                    avatar_url = f\"https://www.guilded.gg/asset/DefaultUserAvatars/profile_{doge[created_by_id]}.png\"\n",
        "\n",
        "                # Process regular message content\n",
        "                content = \"\"\n",
        "                if \"content\" in message and message[\"content\"]:\n",
        "                    document = message[\"content\"].get(\"document\")\n",
        "                    if document:\n",
        "                        nodes = document.get(\"nodes\", [])\n",
        "                        for node in nodes:\n",
        "                            if node.get(\"type\") in (\"image\", \"video\", \"fileUpload\"):\n",
        "                                file_src = node.get(\"data\", {}).get(\"src\", \"\")\n",
        "                                if file_src:\n",
        "                                    parsed_url = urlparse(file_src)\n",
        "                                    media_url = f\"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}\"\n",
        "\n",
        "                                    if node.get(\"type\") == \"fileUpload\":\n",
        "                                        filename = node.get(\"data\", {}).get(\"name\")\n",
        "                                        folder = \"files\"\n",
        "                                    elif node.get(\"type\") == \"image\":\n",
        "                                        filename = os.path.basename(parsed_url.path)\n",
        "                                        folder = \"images\"\n",
        "                                    elif node.get(\"type\") == \"video\":\n",
        "                                        filename = os.path.basename(parsed_url.path)\n",
        "                                        folder = \"videos\"\n",
        "\n",
        "                                    filepath = os.path.join(channel_dir, \"media\", folder, filename)\n",
        "                                    futures.append(executor.submit(download_file, media_url, filepath))\n",
        "\n",
        "                                    if node.get(\"type\") == \"image\":\n",
        "                                        content += f'<img src=\"{os.path.join(\"media\", \"images\", filename)}\">'\n",
        "                                    elif node.get(\"type\") == \"video\":\n",
        "                                        content += f'<video controls><source src=\"{os.path.join(\"media\", \"videos\", filename)}\" type=\"video/webm\">Your browser does not support the video tag.</video>'\n",
        "                                    elif node.get(\"type\") == \"fileUpload\":\n",
        "                                        file_size_bytes = node.get(\"data\", {}).get(\"fileSizeBytes\")\n",
        "                                        file_size_kb = round(file_size_bytes / 1024, 1)\n",
        "                                        content += f'''\n",
        "                                            <div class=\"file-upload-container\">\n",
        "                                                <div class=\"file-icon icon icon-file-attach\"></div>\n",
        "                                                <div class=\"file-info\">\n",
        "                                                    <a href=\"{os.path.join(\"media\", \"files\", filename)}\" download class=\"file-name\">{filename}</a>\n",
        "                                                    <div class=\"file-size\">{file_size_kb} kb</div>\n",
        "                                                </div>\n",
        "                                            </div>\n",
        "                                        '''\n",
        "                                    content += '<br>'\n",
        "\n",
        "\n",
        "                            if node.get(\"type\") == \"code-container\":\n",
        "                                code_lines = node.get(\"nodes\", [])\n",
        "                                code_content = ''\n",
        "                                for code_line in code_lines:\n",
        "                                    if code_line.get(\"type\") == \"code-line\":\n",
        "                                        code_line_nodes = code_line.get(\"nodes\", [])\n",
        "                                        for code_line_node in code_line_nodes:\n",
        "                                            leaves = code_line_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                code_content += leaf_text + '\\n'\n",
        "                                language = node.get(\"data\", {}).get(\"language\", \"\")\n",
        "                                if language:\n",
        "                                    content += f'<pre><code class=\"language-{language}\">{code_content.strip()}</code></pre>'\n",
        "                                else:\n",
        "                                    content += f'<pre><code>{code_content.strip()}</code></pre>'\n",
        "\n",
        "\n",
        "                            elif node.get(\"type\") == \"paragraph\":\n",
        "                                sub_nodes = node.get(\"nodes\", [])\n",
        "                                for sub_node in sub_nodes:\n",
        "                                    if sub_node.get(\"type\") == \"reaction\":\n",
        "                                        reaction_data = sub_node.get(\"data\", {})\n",
        "                                        if reaction_data:\n",
        "                                            reaction = reaction_data.get(\"reaction\")\n",
        "                                            if reaction:\n",
        "                                                reaction_id = reaction.get(\"id\")\n",
        "                                                if reaction_id:\n",
        "                                                    if str(reaction_id).startswith(\"9\") and len(str(reaction_id)) == 8:\n",
        "                                                        reaction_name = reaction.get(\"name\")\n",
        "                                                        if reaction_name:\n",
        "                                                            reaction_emoji = emoji.emojize(f\":{reaction_name}:\", language='alias')\n",
        "                                                            content += reaction_emoji\n",
        "                                                    else:\n",
        "                                                        emoji_dict = next((item for item in emotes_dict.values() if item['id'] == reaction_id), None)\n",
        "                                                        if emoji_dict:\n",
        "                                                            content += f'<img src=\"../../emotes/{emoji_dict[\"id\"]}-{emoji_dict[\"name\"]}.webp\" alt=\"{emoji_dict[\"name\"]}\" title=\"{emoji_dict[\"name\"]}\" class=\"emote\">'\n",
        "                                                        else:\n",
        "                                                            custom_reaction = reaction.get(\"customReaction\")\n",
        "                                                            if isinstance(custom_reaction, bool):\n",
        "                                                                custom_reaction = None\n",
        "                                                            if custom_reaction and \"webp\" in custom_reaction:\n",
        "                                                                emote_id = custom_reaction[\"id\"]\n",
        "                                                                emote_name = custom_reaction[\"name\"]\n",
        "                                                                emote_url = custom_reaction[\"webp\"]\n",
        "                                                                emote_filename = f'{emote_id}-{emote_name}'\n",
        "                                                                emotes_external_dir = os.path.join(server_dir, \"emotes\", \"external\")\n",
        "                                                                if not os.path.exists(emotes_external_dir):\n",
        "                                                                    os.makedirs(emotes_external_dir)\n",
        "                                                                emote_filepath = os.path.join(emotes_external_dir, emote_filename + \".webp\")\n",
        "                                                                if not os.path.exists(emote_filepath):\n",
        "                                                                    response = requests.get(emote_url)\n",
        "                                                                    if response.status_code == 200:\n",
        "                                                                        with open(emote_filepath, \"wb\") as f:\n",
        "                                                                            f.write(response.content)\n",
        "                                                                content += f'<img src=\"../../emotes/external/{emote_filename}.webp\" alt=\"{emote_name}.webp\" title=\"{emote_name}.webp\" class=\"emote\">'\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"mention\":\n",
        "                                        mention_data = sub_node.get(\"data\", {}).get(\"mention\", {})\n",
        "                                        mentioned_id = mention_data.get(\"id\")\n",
        "                                        mentioned_name = mention_data.get(\"name\")\n",
        "                                        member = next((m for m in members_data['members'] if m['id'] == mentioned_id), None)\n",
        "                                        current_name = member['name'] if member else mentioned_name\n",
        "                                        display_name = f'@{mentioned_name}'\n",
        "                                        content += display_name\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "                                    #elif sub_node.get(\"type\") == \"mention\":\n",
        "                                    #    mention_data = sub_node.get(\"data\", {}).get(\"mention\", {})\n",
        "                                    #    mentioned_id = mention_data.get(\"id\")\n",
        "                                    #    mentioned_name = mention_data.get(\"name\")\n",
        "                                    #    member = next((m for m in members_data['members'] if m['id'] == mentioned_id), None)\n",
        "                                    #    current_name = member['name'] if member else mentioned_name\n",
        "                                    #    color = None\n",
        "                                    #    if member and 'roleIds' in member:\n",
        "                                    #        for role_id in member['roleIds']:\n",
        "                                    #            role = roles_data.get(str(role_id))\n",
        "                                    #            if role and role['color'] != 'transparent':\n",
        "                                    #                color = role['color']\n",
        "                                    #                break\n",
        "                                    #    display_name = f'@{mentioned_name}'\n",
        "                                    #    if current_name != mentioned_name:\n",
        "                                    #        realname = f'Current name: @{current_name}'\n",
        "                                    #        if color:\n",
        "                                    #            content += f'<span style=\"color: {color};\" title=\"{realname}\">{display_name}</span>'\n",
        "                                    #        else:\n",
        "                                    #            content += f'<span title=\"{realname}\">{display_name}</span>'\n",
        "                                    #    else:\n",
        "                                    #        if color:\n",
        "                                    #            content += f'<span style=\"color: {color};\">{realname}</span>'\n",
        "                                    #        else:\n",
        "                                    #            content += display_name\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"link\":\n",
        "                                        link_data = sub_node.get(\"data\", {})\n",
        "                                        href = link_data.get(\"href\", \"\")\n",
        "                                        if href:\n",
        "                                            link_text = sub_node.get(\"nodes\", [{}])[0].get(\"leaves\", [{}])[0].get(\"text\", \"\")\n",
        "                                            content += f'<a href=\"{href}\">{link_text}</a>'\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"block-quote-container\":\n",
        "                                        block_quote_lines = sub_node.get(\"nodes\", [])\n",
        "                                        block_quote_content = ''\n",
        "                                        for block_quote_line in block_quote_lines:\n",
        "                                            if block_quote_line.get(\"type\") == \"block-quote-line\":\n",
        "                                                block_quote_line_nodes = block_quote_line.get(\"nodes\", [])\n",
        "                                                for block_quote_line_node in block_quote_line_nodes:\n",
        "                                                    leaves = block_quote_line_node.get(\"leaves\", [])\n",
        "                                                    for leaf in leaves:\n",
        "                                                        leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                        marks = leaf.get(\"marks\", [])\n",
        "                                                        for mark in marks:\n",
        "                                                            mark_type = mark.get(\"type\")\n",
        "                                                            if mark_type == \"inline-code-v2\":\n",
        "                                                                leaf_text = f'<code>{leaf_text}</code>'\n",
        "                                                            elif mark_type == \"bold\":\n",
        "                                                                leaf_text = f'<strong>{leaf_text}</strong>'\n",
        "                                                            elif mark_type == \"italic\":\n",
        "                                                                leaf_text = f'<em>{leaf_text}</em>'\n",
        "                                                            elif mark_type == \"underline\":\n",
        "                                                                leaf_text = f'<u>{leaf_text}</u>'\n",
        "                                                            elif mark_type == \"strikethrough\":\n",
        "                                                                leaf_text = f'<del>{leaf_text}</del>'\n",
        "                                                            elif mark_type == \"spoiler\":\n",
        "                                                                leaf_text = f'<span style=\"background-color: #000; color: #fff;\">{leaf_text}</span>'\n",
        "                                                        block_quote_content += leaf_text\n",
        "                                                block_quote_content += '<br>'\n",
        "                                        content += f'<div style=\"background-color: #f0f0f0; border-left: 4px solid #F5C400; padding: 10px;\">{block_quote_content}</div>'\n",
        "\n",
        "                                    else:\n",
        "                                        leaves = sub_node.get(\"leaves\", [])\n",
        "                                        for leaf in leaves:\n",
        "                                            leaf_text = leaf.get(\"text\", \"\")\n",
        "                                            marks = leaf.get(\"marks\", [])\n",
        "                                            for mark in marks:\n",
        "                                                mark_type = mark.get(\"type\")\n",
        "                                                if mark_type == \"inline-code-v2\":\n",
        "                                                    if mark.get(\"object\") == \"mark\":\n",
        "                                                        leaf_text = sub_node.get(\"leaves\", [{}])[0].get(\"text\", \"\")\n",
        "                                                        leaf_text = leaf_text.replace(\"<script>\", \"\").replace(\"</script>\", \"\")\n",
        "                                                    leaf_text = f'<code>{leaf_text}</code>'\n",
        "                                                elif mark_type == \"bold\":\n",
        "                                                    leaf_text = f'<strong>{leaf_text}</strong>'\n",
        "                                                elif mark_type == \"italic\":\n",
        "                                                    leaf_text = f'<em>{leaf_text}</em>'\n",
        "                                                elif mark_type == \"underline\":\n",
        "                                                    leaf_text = f'<u>{leaf_text}</u>'\n",
        "                                                elif mark_type == \"strikethrough\":\n",
        "                                                    leaf_text = f'<del>{leaf_text}</del>'\n",
        "                                                elif mark_type == \"spoiler\":\n",
        "                                                    leaf_text = f'<span style=\"background-color: #000; color: #fff;\">{leaf_text}</span>'\n",
        "                                            content += leaf_text\n",
        "                                content += '<br>'\n",
        "\n",
        "                            elif node.get(\"type\") == \"channel\":\n",
        "                                chan_nodes = node.get(\"nodes\", [])\n",
        "                                #for chan_node in chan_nodes:\n",
        "                                #    leaves = chan_node.get(\"leaves\", [])\n",
        "                                #    for leaf in leaves:\n",
        "                                #        channel_text = leaf.get(\"text\", \"\")\n",
        "                                #        content += f'{channel_text}'\n",
        "                                channel_data = node.get(\"data\", {}).get(\"channel\", {})\n",
        "                                channel_name = channel_data.get(\"name\", \"\")\n",
        "                                if channel_name:\n",
        "                                    content += f'#{channel_name}'\n",
        "\n",
        "\n",
        "                            elif node.get(\"type\") == \"block-quote-container\":\n",
        "                                block_quote_lines = node.get(\"nodes\", [])\n",
        "                                block_quote_content = ''\n",
        "                                for block_quote_line in block_quote_lines:\n",
        "                                    if block_quote_line.get(\"type\") == \"block-quote-line\":\n",
        "                                        block_quote_line_nodes = block_quote_line.get(\"nodes\", [])\n",
        "                                        for block_quote_line_node in block_quote_line_nodes:\n",
        "                                            leaves = block_quote_line_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                block_quote_content += '<br>' + leaf_text.lstrip('>').strip()\n",
        "                                content += f'<blockquote style=\"border-left: 4px solid #F5C400; padding: 4px;\">{block_quote_content.strip()}</blockquote>'\n",
        "\n",
        "                            elif node.get(\"type\") == \"unordered-list\":\n",
        "                                list_items = node.get(\"nodes\", [])\n",
        "                                unordered_list_content = ''\n",
        "                                for list_item in list_items:\n",
        "                                    if list_item.get(\"type\") == \"list-item\":\n",
        "                                        list_item_nodes = list_item.get(\"nodes\", [])\n",
        "                                        list_item_content = ''\n",
        "                                        for list_item_node in list_item_nodes:\n",
        "                                            leaves = list_item_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                list_item_content += leaf_text\n",
        "                                        unordered_list_content += f'<li>{list_item_content}</li>'\n",
        "                                content += f'<ul>{unordered_list_content}</ul>'\n",
        "                            elif node.get(\"type\") == \"ordered-list\":\n",
        "                                list_items = node.get(\"nodes\", [])\n",
        "                                ordered_list_content = ''\n",
        "                                for list_item in list_items:\n",
        "                                    if list_item.get(\"type\") == \"list-item\":\n",
        "                                        list_item_nodes = list_item.get(\"nodes\", [])\n",
        "                                        list_item_content = ''\n",
        "                                        for list_item_node in list_item_nodes:\n",
        "                                            leaves = list_item_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                list_item_content += leaf_text\n",
        "                                        ordered_list_content += f'<li>{list_item_content}</li>'\n",
        "                                    elif list_item.get(\"type\") == \"ordered-list\":\n",
        "                                        nested_ordered_list_content = ''\n",
        "                                        nested_list_items = list_item.get(\"nodes\", [])\n",
        "                                        for nested_list_item in nested_list_items:\n",
        "                                            if nested_list_item.get(\"type\") == \"list-item\":\n",
        "                                                nested_list_item_nodes = nested_list_item.get(\"nodes\", [])\n",
        "                                                nested_list_item_content = ''\n",
        "                                                for nested_list_item_node in nested_list_item_nodes:\n",
        "                                                    nested_leaves = nested_list_item_node.get(\"leaves\", [])\n",
        "                                                    for nested_leaf in nested_leaves:\n",
        "                                                        nested_leaf_text = nested_leaf.get(\"text\", \"\")\n",
        "                                                        nested_list_item_content += nested_leaf_text\n",
        "                                                nested_ordered_list_content += f'<li>{nested_list_item_content}</li>'\n",
        "                                        ordered_list_content += f'<ol>{nested_ordered_list_content}</ol>'\n",
        "                                content += f'<ol>{ordered_list_content}</ol>'\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            formatted_message = MESSAGE_TEMPLATE.format(\n",
        "                avatar_url=avatar_url,\n",
        "                created_by=created_by,\n",
        "                created_at=created_at,\n",
        "                content=content\n",
        "            )\n",
        "            formatted_messages.append(formatted_message)\n",
        "\n",
        "        # Wait for all download tasks to complete\n",
        "        for future in as_completed(futures):\n",
        "            future.result()\n",
        "\n",
        "    html = HTML_TEMPLATE.format(\n",
        "        title=channel_data[\"name\"],\n",
        "        topic=channel_data.get(\"topic\", \"\"),\n",
        "        messages='\\n'.join(formatted_messages),\n",
        "        CSS=CSS\n",
        "    )\n",
        "\n",
        "    return html\n",
        "\n",
        "def get_all_src_values(obj):\n",
        "    src_values = []\n",
        "\n",
        "    if isinstance(obj, dict):\n",
        "        for key, value in obj.items():\n",
        "            if key == \"src\":\n",
        "                src_values.append(value)\n",
        "            else:\n",
        "                src_values.extend(get_all_src_values(value))\n",
        "    elif isinstance(obj, list):\n",
        "        for item in obj:\n",
        "            src_values.extend(get_all_src_values(item))\n",
        "\n",
        "    return src_values\n",
        "\n",
        "# Process each channel\n",
        "for channel in channels_data[\"channels\"]:\n",
        "    channel_dir = os.path.join(server_dir, sanitize_filename(channel[\"name\"]))\n",
        "    for group in groups_data[\"groups\"]:\n",
        "        if channel[\"groupId\"] == group[\"id\"]:\n",
        "            group_dir = os.path.join(server_dir, group[\"name\"] + \" (\" + group[\"id\"] + \")\")\n",
        "            channel_dir = os.path.join(group_dir, sanitize_filename(channel[\"name\"]))\n",
        "            break\n",
        "\n",
        "    if channel.get(\"contentType\") in [\"chat\", \"stream\", \"voice\"]: # DONE. Todo maybe fix system stuff and add custom member dict but idc\n",
        "        if not doChats:\n",
        "            continue\n",
        "        print(f\"=========== FETCHING MESSAGES FROM #{channel['name']} in {group['name']}...\")\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        messages = []\n",
        "        beforeDate = None\n",
        "        mesc = 0\n",
        "        while True:\n",
        "            params = {\"limit\": 100}\n",
        "            if beforeDate:\n",
        "                params[\"beforeDate\"] = beforeDate\n",
        "            response = fetch(f\"channels/{channel['id']}/messages\", params=params)\n",
        "            messages.extend(response[\"messages\"])\n",
        "            mesc += len(response[\"messages\"])\n",
        "            #print(f\"Collected {mesc} messages\")\n",
        "            if len(response[\"messages\"]) < 100:\n",
        "                break\n",
        "            beforeDate = response[\"messages\"][-1][\"createdAt\"]\n",
        "        messages.reverse()\n",
        "        # Save messages JSON\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_messages.json\"), 'w') as f:\n",
        "            json.dump(messages, f, separators=(',', ':'))\n",
        "        if verbose_mescount:\n",
        "            print(f\"Found {len(messages)} messages\")\n",
        "            print(f\"Saved all messages from #{channel['name']} to {channel_dir}/{channel['id']}_messages.json\")\n",
        "        # Generate and save HTML\n",
        "        if len(messages) > messages_per_page:\n",
        "            for i in range(0, len(messages), messages_per_page):\n",
        "                html = generate_html(messages[i:i+messages_per_page], channel, members_json, server_dir, channel_dir)\n",
        "                html_filename = f\"{channel['name']} - Page {i//messages_per_page + 1}.html\".replace('/', '-')\n",
        "                with open(os.path.join(channel_dir, html_filename), 'w') as f:\n",
        "                    f.write(html)\n",
        "                if verbose_mescount:\n",
        "                    print(f\"Saved {channel_dir}/{html_filename}\")\n",
        "        else:\n",
        "            html = generate_html(messages, channel, members_json, server_dir, channel_dir)\n",
        "            html_filename = f\"{channel['name']}.html\".replace('/', '-')\n",
        "            with open(os.path.join(channel_dir, html_filename), 'w') as f:\n",
        "                f.write(html)\n",
        "            if verbose_mescount:\n",
        "                print(f\"Saved {channel_dir}/{html_filename}\")\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"event\": #DONE. Too lazy to csv\n",
        "        if not doEvents:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        cal = fetch(f\"channels/{channel['id']}/events\")\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_calendar.json\"), 'w') as f:\n",
        "            json.dump(cal, f, separators=(',', ':'))\n",
        "            if verbose_mescount:\n",
        "                print(f\"saved {channel_dir}/{channel['id']}_calendar.json\")\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"scheduling\": #DONE. Too lazy to csv\n",
        "        if not doSchedules:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        calurl = f\"{guilded}/channels/{channel['id']}/availability\"\n",
        "        cal = requests.get(calurl, cookies=cookies)\n",
        "        caler = cal.json()\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_schedule.json\"), 'w') as f:\n",
        "            json.dump(caler, f, separators=(',', ':'))\n",
        "            if verbose_mescount:\n",
        "                print(f\"saved {channel_dir}/{channel['id']}_schedule.json\")\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"media\": # DONE. I didn't check for same names but idm it like this personally\n",
        "        if not doMedia:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        print(f\"=========== FETCHING MEDIA FROM #{channel['name']} in {group['name']}...\")\n",
        "        medias = []\n",
        "        beforeDate = None\n",
        "        while True:\n",
        "            mediaurl = f\"{guilded}/channels/{channel['id']}/media\"\n",
        "            params = {\"pageSize\": 40}\n",
        "            if beforeDate:\n",
        "                params[\"beforeDate\"] = beforeDate\n",
        "            response = requests.get(mediaurl, params=params, cookies=cookies)\n",
        "            response.raise_for_status()\n",
        "            data = response.json()\n",
        "            for item in data:\n",
        "                if \"src\" in item:\n",
        "                    item[\"src\"] = shit(item[\"src\"])\n",
        "            medias.extend(data)\n",
        "            if len(data) < 40:\n",
        "                break\n",
        "            if data:\n",
        "                beforeDate = data[-1][\"createdAt\"]\n",
        "            else:\n",
        "                break\n",
        "        for item in medias:\n",
        "            filename = os.path.basename(item[\"src\"]).split('?')[0]\n",
        "            mname = sanitize_filename(item[\"description\"])\n",
        "            mid = f\"{item['id']}\"\n",
        "            midpath = os.path.join(channel_dir, mname)\n",
        "            os.makedirs(midpath, exist_ok=True)\n",
        "            filepath = os.path.join(midpath, filename)\n",
        "            with open(os.path.join(midpath, f\"{mid}.json\"), 'w') as f:\n",
        "                json.dump(item, f, separators=(',', ':'))\n",
        "            lmao(item[\"src\"], filepath, cookies)\n",
        "            repliesurl = f\"{guilded}/content/team_media/{mid}/replies\"\n",
        "            response = requests.get(repliesurl, cookies=cookies)\n",
        "            response.raise_for_status()\n",
        "            replies = response.json()\n",
        "            if replies:\n",
        "                with open(os.path.join(midpath, f\"{mid}_replies.json\"), 'w') as f:\n",
        "                    json.dump(replies, f, separators=(',', ':'))\n",
        "                replies_folder = os.path.join(midpath, \"replies\")\n",
        "                os.makedirs(replies_folder, exist_ok=True)\n",
        "                for reply in replies:\n",
        "                    if \"message\" in reply and \"document\" in reply[\"message\"]:\n",
        "                        for node in reply[\"message\"][\"document\"][\"nodes\"]:\n",
        "                            if node[\"type\"] == \"image\" and \"data\" in node and \"src\" in node[\"data\"]:\n",
        "                                reply_image_url = shit(node[\"data\"][\"src\"])\n",
        "                                reply_filename = os.path.basename(reply_image_url).split('?')[0]\n",
        "                                reply_filepath = os.path.join(replies_folder, reply_filename)\n",
        "                                lmao(reply_image_url, reply_filepath, cookies)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_media.json\"), 'w') as f:\n",
        "            json.dump(medias, f, separators=(',', ':'))\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"doc\": #DONE. Think we got all replies. I didn't check for same names but idm it like this personally\n",
        "        if not doDocs:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        print(f\"=========== FETCHING DOCS FROM #{channel['name']} in {group['name']}...\")\n",
        "\n",
        "        docs = []\n",
        "        docs_dir = os.path.join(channel_dir, \"docs\")\n",
        "        os.makedirs(docs_dir, exist_ok=True)\n",
        "\n",
        "        docurl = f\"{guilded}/channels/{channel['id']}/docs\"\n",
        "        params = {\"maxItems\": 50}\n",
        "        response = requests.get(docurl, params=params, cookies=cookies)\n",
        "        response.raise_for_status()\n",
        "\n",
        "        data = response.json()\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_docs.json\"), 'w') as f:\n",
        "            json.dump(data, f, separators=(',', ':'))\n",
        "        for item in data:\n",
        "            title = sanitize_filename(item[\"title\"])\n",
        "            item_dir = os.path.join(docs_dir, title)\n",
        "            os.makedirs(item_dir, exist_ok=True)\n",
        "            fulldoc = requests.get(f\"{docurl}/{item['id']}\", cookies=cookies)\n",
        "            docdata = fulldoc.json()\n",
        "            dontcare(docdata)\n",
        "            with open(os.path.join(item_dir, f\"{item['id']}.json\"), 'w') as f:\n",
        "                json.dump(docdata, f, separators=(',', ':'))\n",
        "\n",
        "            if \"content\" in docdata and \"document\" in docdata[\"content\"]:\n",
        "                for node in docdata[\"content\"][\"document\"][\"nodes\"]:\n",
        "                    if node[\"type\"] == \"image\" and \"data\" in node:\n",
        "                        upload = node[\"data\"]\n",
        "                        if \"name\" in upload and \"src\" in upload:\n",
        "                            original_filename = upload[\"name\"]  # This is the name of the file\n",
        "                            download_url = upload[\"src\"]  # This is the URL to download the file from\n",
        "                            parsed_url = urllib.parse.urlparse(download_url)\n",
        "                            url_path = parsed_url.path\n",
        "                            _, url_extension = os.path.splitext(url_path)\n",
        "                            filename = f\"{original_filename}{url_extension}\"\n",
        "                            saveto = os.path.join(docs_dir, title, filename)\n",
        "                            try:\n",
        "                                response = requests.get(download_url, cookies=cookies)\n",
        "                                response.raise_for_status()\n",
        "                                with open(saveto, \"wb\") as f:\n",
        "                                    f.write(response.content)\n",
        "                                if verbose_mescount:\n",
        "                                    print(f\"Downloaded {filename} from {download_url}\")\n",
        "                            except requests.RequestException as e:\n",
        "                                print(f\"Error downloading {filename} from {download_url}: {e}\")\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"announcement\": #DONE, think we got replies too\n",
        "        if not doAnnouncements:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        print(f\"=========== FETCHING ANNOUNCEMENTS FROM #{channel['name']} in {group['name']}...\")\n",
        "        annurl = f\"{guilded}/channels/{channel['id']}/announcements\"\n",
        "        anns = []\n",
        "        params = {\"maxItems\": 3}\n",
        "        beforeDate = None\n",
        "        while True:\n",
        "            if beforeDate:\n",
        "                params[\"beforeDate\"] = beforeDate\n",
        "            response = requests.get(annurl, params=params, cookies=cookies)\n",
        "            response.raise_for_status()\n",
        "            data = unshid_cdn(response.json())\n",
        "            anns.extend(data[\"announcements\"])\n",
        "            if len(data[\"announcements\"]) < 3:\n",
        "                break\n",
        "            if data[\"announcements\"]:\n",
        "                beforeDate = data[\"announcements\"][-1][\"createdAt\"]\n",
        "            else:\n",
        "                break\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_announcements.json\"), 'w') as f:\n",
        "            json.dump(anns, f, separators=(',', ':'))\n",
        "        for ann in anns:\n",
        "            annid = ann[\"id\"]\n",
        "            title = ann[\"title\"]\n",
        "            annpath = os.path.join(channel_dir, title)\n",
        "            os.makedirs(annpath, exist_ok=True)\n",
        "            with open(os.path.join(annpath, f\"{annid}.json\"), 'w') as f:\n",
        "                json.dump(ann, f, separators=(',', ':'))\n",
        "\n",
        "            links = []\n",
        "            def find_links(data):\n",
        "                if isinstance(data, dict):\n",
        "                    if \"src\" in data:\n",
        "                        links.append(data[\"src\"])\n",
        "                    else:\n",
        "                        for value in data.values():\n",
        "                            find_links(value)\n",
        "                elif isinstance(data, list):\n",
        "                    for item in data:\n",
        "                        find_links(item)\n",
        "            find_links(ann[\"content\"][\"document\"][\"nodes\"])\n",
        "            mediadir = os.path.join(annpath, \"media\")\n",
        "            for link in links:\n",
        "                url = link\n",
        "                os.makedirs(mediadir, exist_ok=True)\n",
        "                parsed_url = urllib.parse.urlparse(url)\n",
        "                filename = os.path.basename(parsed_url.path)\n",
        "                filepath = os.path.join(mediadir, filename)\n",
        "                response = requests.get(url)\n",
        "                with open(filepath, 'wb') as f:\n",
        "                    f.write(response.content)\n",
        "                    if verbose_mescount:\n",
        "                        print(f\"Downloaded {filepath}\")\n",
        "\n",
        "            annrep = f\"{guilded}/content/announcement/{annid}/replies\"\n",
        "            rep_response = requests.get(annrep, cookies=cookies).json()\n",
        "\n",
        "            with open(os.path.join(annpath, f\"{annid}_replies.json\"), 'w') as f:\n",
        "                json.dump(rep_response, f, separators=(',', ':'))\n",
        "\n",
        "            links = re.findall(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\\\(\\\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', str(rep_response))\n",
        "            links = [re.sub(r'^https://s3-us-west-2\\.amazonaws\\.com/www\\.guilded\\.gg/', 'https://cdn.gilcdn.com/', url) for url in links]\n",
        "            for url in links:\n",
        "                os.makedirs(mediadir, exist_ok=True)\n",
        "                response = requests.get(url, stream=True)\n",
        "                if response.status_code == 200:\n",
        "                    filename = os.path.basename(response.url).split('?')[0]\n",
        "                    filepath = os.path.join(mediadir, filename)\n",
        "                    with open(filepath, 'wb') as f:\n",
        "                        for chunk in response.iter_content(chunk_size=1024):\n",
        "                            if chunk:\n",
        "                                f.write(chunk)\n",
        "                        if verbose_mescount:\n",
        "                            print (f\"Downloaded {filepath}\")\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"list\": # DONE, no need to process.\n",
        "        if not doLists:\n",
        "            continue\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        print(f\"=========== FETCHING TODO FROM #{channel['name']} in {group['name']}...\")\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        todourl = f\"{guilded}/channels/{channel['id']}/listitems\"\n",
        "        response = requests.get(todourl, cookies=cookies)\n",
        "        response.raise_for_status()\n",
        "        todo = response.json()\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_todo.json\"), 'w') as f:\n",
        "            json.dump(todo, f, separators=(',', ':'))\n",
        "\n",
        "\n",
        "    if channel.get(\"contentType\") == \"forum\": # DOOOOONE. May we all love one day.\n",
        "        if not doForums:\n",
        "            continue\n",
        "        print(f\"=========== FETCHING THREADS FROM #{channel['name']}...\")\n",
        "        os.makedirs(channel_dir, exist_ok=True)\n",
        "        with open(os.path.join(channel_dir, f\"{channel['id']}_info.json\"), 'w') as f:\n",
        "            json.dump(channel, f, separators=(',', ':'))\n",
        "        all_threads = []\n",
        "        page = 1\n",
        "\n",
        "        while True:\n",
        "            forumurl = f\"{guilded}/channels/{channel['id']}/forums\"\n",
        "            params = {\"maxItems\": 30, \"page\": page}\n",
        "\n",
        "            response = requests.get(forumurl, params=params, cookies=cookies)\n",
        "            response.raise_for_status()\n",
        "            data = unshid_cdn(response.json())\n",
        "\n",
        "            threads = data.get(\"threads\", [])\n",
        "            if not threads:\n",
        "                break\n",
        "            all_threads.extend(threads)\n",
        "            page += 1\n",
        "\n",
        "            with open(os.path.join(channel_dir, f\"{channel['id']}_threads.json\"), 'w') as f:\n",
        "                json.dump(threads, f, separators=(',', ':'))\n",
        "\n",
        "\n",
        "\n",
        "            for thread in data.get(\"threads\", []):\n",
        "                thread_id = thread[\"id\"]\n",
        "                thread_title = sanitize_filename(thread[\"title\"])\n",
        "                thread_dir = os.path.join(channel_dir, thread_title)\n",
        "                os.makedirs(thread_dir, exist_ok=True)\n",
        "                if verbose_mescount:\n",
        "                    print(f\"Fetching {thread_id}\")\n",
        "                thread_url = f\"{guilded}/channels/{channel['id']}/forums/{thread_id}\"\n",
        "                thread_response = unshid_cdn(requests.get(thread_url, cookies=cookies).json())\n",
        "                os.makedirs(os.path.join(channel_dir, \"threads\"), exist_ok=True)\n",
        "                with open(os.path.join(thread_dir, f\"{thread_id}_thread.json\"), 'w') as f:\n",
        "                    json.dump(thread_response, f, separators=(',', ':'))\n",
        "                media_dir = os.path.join(thread_dir, \"media\")\n",
        "\n",
        "\n",
        "                afterDate = None\n",
        "                postc = 0\n",
        "                posts = []\n",
        "                while True:\n",
        "                    params = {\"maxItems\": 50}\n",
        "                    if afterDate:\n",
        "                        params[\"afterDate\"] = afterDate\n",
        "\n",
        "                    response = fetch(f\"channels/{channel['id']}/forums/{thread_id}/replies\", params=params)\n",
        "                    posts.extend(response[\"threadReplies\"])\n",
        "\n",
        "                    # Check if fewer than 50 replies were returned, indicating the last page\n",
        "                    if len(response[\"threadReplies\"]) < 50:\n",
        "                        break\n",
        "\n",
        "                    afterDate = response[\"threadReplies\"][-1][\"createdAt\"]\n",
        "\n",
        "                # Write the complete posts to a JSON file\n",
        "                with open(os.path.join(thread_dir, f\"{thread_id}_replies.json\"), 'w') as f:\n",
        "                    json.dump(posts, f, separators=(',', ':'))\n",
        "\n",
        "\n",
        "                # This is terrible.\n",
        "                os.makedirs(media_dir, exist_ok=True)\n",
        "\n",
        "                for post in posts:\n",
        "                    if \"message\" in post and \"document\" in post[\"message\"]:\n",
        "                        for node in post[\"message\"][\"document\"][\"nodes\"]:\n",
        "                            if node[\"object\"] == \"block\":\n",
        "                                if node[\"type\"] == \"fileUpload\" and \"data\" in node:\n",
        "                                    file_url = node[\"data\"][\"src\"]\n",
        "                                    realname = os.path.splitext(node[\"data\"][\"name\"])[0]\n",
        "                                    up_name = unquote(os.path.basename(urlparse(file_url).path)).replace(\"-Full\", \"\")\n",
        "                                    file_name = f\"{realname}_{up_name}\"\n",
        "                                    file_path = os.path.join(media_dir, file_name)\n",
        "                                elif node[\"type\"] in [\"image\", \"video\"] and \"data\" in node:\n",
        "                                    file_url = node[\"data\"][\"src\"]\n",
        "                                    parsed_url = urlparse(file_url)\n",
        "                                    url_path = parsed_url.path\n",
        "                                    _, url_extension = os.path.splitext(url_path)\n",
        "                                    file_name = f\"{unquote(os.path.basename(url_path))}\"\n",
        "                                    file_path = os.path.join(media_dir, file_name)\n",
        "                                else:\n",
        "                                    continue\n",
        "\n",
        "                                try:\n",
        "                                    media_response = requests.get(file_url)\n",
        "                                    media_response.raise_for_status()\n",
        "                                    with open(file_path, 'wb') as f:\n",
        "                                        f.write(media_response.content)\n",
        "                                    if verbose_mescount:\n",
        "                                        print(f\"Downloaded {file_name} from {file_url}\")\n",
        "                                except requests.RequestException as e:\n",
        "                                    print(f\"Error downloading {file_name} from {file_url}: {e}\")\n",
        "\n",
        "\n",
        "                for node in thread_response[\"thread\"][\"message\"][\"document\"][\"nodes\"]:\n",
        "                    if node[\"object\"] == \"block\":\n",
        "                        if node[\"type\"] == \"fileUpload\" and \"data\" in node:\n",
        "                            file_url = node[\"data\"][\"src\"]\n",
        "                            realname = os.path.splitext(node[\"data\"][\"name\"])[0]\n",
        "                            up_name = unquote(os.path.basename(urlparse(file_url).path)).replace(\"-Full\", \"\")\n",
        "                            file_name = f\"{realname}_{up_name}\"\n",
        "                            file_path = os.path.join(media_dir, file_name)\n",
        "                        elif node[\"type\"] in [\"image\", \"video\"] and \"data\" in node:\n",
        "                            file_url = node[\"data\"][\"src\"]\n",
        "                            parsed_url = urlparse(file_url)\n",
        "                            url_path = parsed_url.path\n",
        "                            _, url_extension = os.path.splitext(url_path)\n",
        "                            file_name = f\"{unquote(os.path.basename(url_path))}\"\n",
        "                            file_path = os.path.join(media_dir, file_name)\n",
        "                        else:\n",
        "                            continue\n",
        "\n",
        "                        try:\n",
        "                            media_response = requests.get(file_url)\n",
        "                            media_response.raise_for_status()\n",
        "                            with open(file_path, 'wb') as f:\n",
        "                                f.write(media_response.content)\n",
        "                            if verbose_mescount:\n",
        "                                print(f\"Downloaded {file_name} from {file_url}\")\n",
        "                        except requests.RequestException as e:\n",
        "                            print(f\"Error downloading {file_name} from {file_url}: {e}\")\n",
        "\n",
        "\n",
        "for group in groups_data[\"groups\"]:\n",
        "    group_dir = os.path.join(server_dir, group[\"name\"] + \" (\" + group[\"id\"] + \")\")\n",
        "    if group.get(\"avatar\"):\n",
        "        parsed_url = urlparse(group[\"avatar\"]) # Parse the URL\n",
        "        pic_filename = os.path.basename(parsed_url.path) # Extract filename from parsed URL's path\n",
        "        if not os.path.exists(os.path.join(group_dir, pic_filename)):\n",
        "            os.makedirs(group_dir, exist_ok=True)\n",
        "        pic_filepath = os.path.join(group_dir, pic_filename)\n",
        "        response = requests.get(group[\"avatar\"]) # Pass original URL string to requests.get\n",
        "        if response.status_code == 200:\n",
        "            with open(pic_filepath, \"wb\") as f:\n",
        "                f.write(response.content)\n",
        "\n",
        "print(\"Finished processing all channels.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "VvddyRSTldBW"
      },
      "outputs": [],
      "source": [
        "# @title Download the Whole Server\n",
        "# @markdown You should run this anyway because this will clean lots of redundant empty files and folders.\n",
        "\n",
        "# @markdown If your output archive seems way too big (more than 2GB), DDL will be very unreliable. You should use GDrive instead or find a way to get Colab to send it to you.\n",
        "\n",
        "for dirpath, dirnames, filenames in os.walk(server_dir):\n",
        "  for filename in filenames:\n",
        "    filepath = os.path.join(dirpath, filename)\n",
        "    if os.path.isfile(filepath) and os.stat(filepath).st_size == 2:\n",
        "      with open(filepath, 'r') as f:\n",
        "        content = f.read()\n",
        "        if content == '[]':\n",
        "          os.remove(filepath)\n",
        "          #print(f\"cleaned {filepath}\")\n",
        "  for dirname in dirnames:\n",
        "    dirpath_to_check = os.path.join(dirpath, dirname)\n",
        "    if not os.listdir(dirpath_to_check):\n",
        "      os.rmdir(dirpath_to_check)\n",
        "      #print(f\"cleaned {dirpath_to_check}\")\n",
        "\n",
        "import shutil\n",
        "import subprocess\n",
        "from google.colab import drive\n",
        "from google.colab import files\n",
        "save_type = \"7z\" # @param [\"7z\", \"targz\", \"zip\"]\n",
        "saving_to = \"GoogleDrive\" # @param [\"DirectDownLoad\", \"GoogleDrive\"]\n",
        "todl = SAVE_DIRECTORY + f\"{SERVER_NAME}\"\n",
        "if save_type == \"7z\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}{SERVER_NAME}.7z\"\n",
        "    !7z a -mx0 \"{outfile}\" \"{todl}\"\n",
        "if save_type == \"targz\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}{SERVER_NAME}.tar.gz\"\n",
        "    !tar -czvf \"{outfile}\" \"{todl}\"\n",
        "if save_type == \"zip\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}{SERVER_NAME}.zip\"\n",
        "    !zip -r \"{outfile}\" \"{todl}\"\n",
        "\n",
        "if saving_to == \"DirectDownLoad\":\n",
        "    files.download(outfile)\n",
        "if saving_to == \"GoogleDrive\":\n",
        "    drive.mount('/content/drive')\n",
        "    !cp \"{outfile}\" /content/drive/MyDrive"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tP3TAKorIxm8"
      },
      "source": [
        "# Direct Messages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "c6PtV2HNZJxh"
      },
      "outputs": [],
      "source": [
        "# @title Export all DMs and Media\n",
        "messages_per_page = 15000 # @param {type:\"integer\"}\n",
        "# @markdown Same deal as above, but you can probably increase the message count way more as DMs aren't as scary as servers. This also works on Groups.\n",
        "from urllib.parse import urlparse, unquote\n",
        "# @markdown Make sure to at least tick the HTML File Setup cell if you skipped doing any server stuff.\n",
        "verbose_dmfetch = True # @param {type:\"boolean\"}\n",
        "\n",
        "\n",
        "def download_file(url, filename):\n",
        "    try:\n",
        "        response = requests.get(url, timeout=45)\n",
        "        if response.status_code == 200:\n",
        "            os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
        "            with open(filename, \"wb\") as f:\n",
        "                f.write(response.content)\n",
        "    except requests.exceptions.Timeout:\n",
        "        print(f\"Timeout occurred while downloading {url}. Skipping...\")\n",
        "    except requests.exceptions.RequestException as e:\n",
        "        print(f\"Error downloading {url}: {e}\")\n",
        "\n",
        "# ... rest of the code remains the same ...\n",
        "def generate_html(messages, dm_data, members_data, dms_dir):\n",
        "    formatted_messages = []\n",
        "    doge = {}\n",
        "\n",
        "    with ThreadPoolExecutor() as executor:\n",
        "        futures = []\n",
        "\n",
        "        for message in messages:\n",
        "            created_by_id = message['createdBy']\n",
        "            created_at = message.get('createdAt') or message.get('created_at')\n",
        "\n",
        "            # Determine if it's a system message\n",
        "            is_system_message = message.get('type') == 'system' or 'systemMessage' in str(message.get('content', {}))\n",
        "\n",
        "            if is_system_message:\n",
        "                member = next((m for m in dm_data['users'] if m['id'] == created_by_id), None)\n",
        "                created_by = member[\"name\"] if member else \"[deleted user]\"\n",
        "                avatar_url = \"\"\n",
        "\n",
        "                # Handle system message content\n",
        "                content = \"\"\n",
        "                document = message.get('content', {}).get('document', {})\n",
        "                for node in document.get('nodes', []):\n",
        "                    if node.get('type') == 'systemMessage':\n",
        "                        data = node.get('data', {})\n",
        "                        message_type = data.get('type')\n",
        "                        if message_type == 'team-channel-created':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) created this chat channel.\"\n",
        "                        elif message_type == 'channel-renamed':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) renamed this channel from '{data.get('oldName')}' to '{data.get('newName')}'.\"\n",
        "                        elif message_type == 'streaming-screenshare-started':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) started to share their screen.\"\n",
        "                        elif message_type == 'voice-call-started':\n",
        "                            content = f\"{created_by} ({data.get('createdBy', '[deleted user]')}) started a voice call.\"\n",
        "                        elif message_type == 'webhookMessage':\n",
        "                            embeds = node.get('data', {}).get('embeds', [])\n",
        "                            content = json.dumps(data, indent=2)\n",
        "                        else:\n",
        "                            content = f\"System action performed: {message_type}\"\n",
        "\n",
        "\n",
        "            else:\n",
        "                # Find member data based on createdBy ID\n",
        "                member = next((m for m in dm_data['users'] if m['id'] == created_by_id), None)\n",
        "                created_by = member[\"name\"] if member else \"[deleted user]\"\n",
        "                #print(\"Member data:\", member)  # Print the member data for inspection\n",
        "\n",
        "                if member and member.get('profilePicture'):\n",
        "                    profile_picture_url = member['profilePicture']\n",
        "                    filename = f\"{created_by_id}{os.path.splitext(os.path.basename(unquote(urlparse(profile_picture_url).path)))[1]}\"\n",
        "                    filepath = os.path.join(dm_dir, \"media\", filename)\n",
        "                    os.makedirs(os.path.dirname(filepath), exist_ok=True)\n",
        "                    with open(filepath, \"wb\") as f:\n",
        "                        f.write(requests.get(profile_picture_url).content)\n",
        "                    avatar_url = os.path.join(\"media\", filename)\n",
        "                else:\n",
        "                    avatar_url = f\"https://www.guilded.gg/asset/DefaultUserAvatars/profile_{doge.setdefault(created_by_id, (len(doge) % 5) + 1)}.png\"\n",
        "\n",
        "\n",
        "                # Process regular message content\n",
        "                content = \"\"\n",
        "                if \"content\" in message and message[\"content\"]:\n",
        "                    document = message[\"content\"].get(\"document\")\n",
        "                    if document:\n",
        "                        nodes = document.get(\"nodes\", [])\n",
        "                        for node in nodes:\n",
        "                            if node.get(\"type\") in (\"image\", \"video\", \"fileUpload\"):\n",
        "                                file_src = node.get(\"data\", {}).get(\"src\", \"\")\n",
        "                                if file_src:\n",
        "                                    parsed_url = urlparse(file_src)\n",
        "                                    media_url = f\"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}\"\n",
        "\n",
        "                                    if node.get(\"type\") == \"fileUpload\":\n",
        "                                        file_src = node.get(\"data\", {}).get(\"src\", \"\")\n",
        "                                        if file_src:\n",
        "                                            parsed_url = urlparse(file_src)\n",
        "                                            media_url = f\"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}\"\n",
        "\n",
        "                                            #filename = f\"{node.get('data', {}).get('name')}_{message['id']}\"\n",
        "                                            filename = f\"{node.get('data', {}).get('name')}_{message['id']}{os.path.splitext(os.path.basename(unquote(urlparse(file_src).path)))[1]}\"\n",
        "                                            folder = \"files\"\n",
        "                                            filepath = os.path.join(dm_dir, \"media\", folder, filename)\n",
        "                                            os.makedirs(os.path.dirname(filepath), exist_ok=True)\n",
        "                                            futures.append(executor.submit(download_file, media_url, filepath))\n",
        "\n",
        "\n",
        "\n",
        "                                    elif node.get(\"type\") == \"image\":\n",
        "                                        filename = os.path.basename(parsed_url.path)\n",
        "                                        folder = \"images\"\n",
        "                                    elif node.get(\"type\") == \"video\":\n",
        "                                        filename = os.path.basename(parsed_url.path)\n",
        "                                        folder = \"videos\"\n",
        "\n",
        "                                    filepath = os.path.join(dm_dir, \"media\", folder, filename)\n",
        "                                    futures.append(executor.submit(download_file, media_url, filepath))\n",
        "                                    if verbose_dmfetch:\n",
        "                                        print(f\"{filename}\")\n",
        "\n",
        "                                    if node.get(\"type\") == \"image\":\n",
        "                                        content += f'<img src=\"{os.path.join(\"media\", \"images\", filename)}\">'\n",
        "                                    elif node.get(\"type\") == \"video\":\n",
        "                                        content += f'<video controls><source src=\"{os.path.join(\"media\", \"videos\", filename)}\" type=\"video/webm\">Your browser does not support the video tag.</video>'\n",
        "                                    elif node.get(\"type\") == \"fileUpload\":\n",
        "                                        file_size_bytes = node.get(\"data\", {}).get(\"fileSizeBytes\")\n",
        "                                        file_size_kb = round(file_size_bytes / 1024, 1)\n",
        "                                        content += f'''\n",
        "                                            <div class=\"file-upload-container\">\n",
        "                                                <div class=\"file-icon icon icon-file-attach\"></div>\n",
        "                                                <div class=\"file-info\">\n",
        "                                                    <a href=\"{os.path.join(\"media\", \"files\", filename)}\" download class=\"file-name\">{node.get('data', {}).get('name')}</a>\n",
        "                                                    <div class=\"file-size\">{file_size_kb} kb</div>\n",
        "                                                </div>\n",
        "                                            </div>\n",
        "                                        '''\n",
        "\n",
        "\n",
        "                            if node.get(\"type\") == \"code-container\":\n",
        "                                code_lines = node.get(\"nodes\", [])\n",
        "                                code_content = ''\n",
        "                                for code_line in code_lines:\n",
        "                                    if code_line.get(\"type\") == \"code-line\":\n",
        "                                        code_line_nodes = code_line.get(\"nodes\", [])\n",
        "                                        for code_line_node in code_line_nodes:\n",
        "                                            leaves = code_line_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                code_content += leaf_text + '\\n'\n",
        "                                language = node.get(\"data\", {}).get(\"language\", \"\")\n",
        "                                if language:\n",
        "                                    content += f'<pre><code class=\"language-{language}\">{code_content.strip()}</code></pre>'\n",
        "                                else:\n",
        "                                    content += f'<pre><code>{code_content.strip()}</code></pre>'\n",
        "\n",
        "\n",
        "                            elif node.get(\"type\") == \"paragraph\":\n",
        "                                sub_nodes = node.get(\"nodes\", [])\n",
        "                                for sub_node in sub_nodes:\n",
        "                                    if sub_node.get(\"type\") == \"reaction\":\n",
        "                                        reaction_data = sub_node.get(\"data\", {})\n",
        "                                        if reaction_data and (reaction := reaction_data.get(\"reaction\")) and (reaction_id := reaction.get(\"id\")):\n",
        "                                            if str(reaction_id).startswith(\"9\") and len(str(reaction_id)) == 8:\n",
        "                                                content += emoji.emojize(f\":{reaction.get('name')}:\", language='alias') if reaction.get('name') else ''\n",
        "                                            else:\n",
        "\n",
        "                                                custom_reaction = reaction.get(\"customReaction\") or {}\n",
        "                                                if isinstance(custom_reaction, bool):\n",
        "                                                    custom_reaction = {}\n",
        "                                                emote_url = custom_reaction.get(\"webp\")\n",
        "                                                emote_filename = custom_reaction.get(\"name\")\n",
        "                                                emote_id = custom_reaction.get(\"id\")\n",
        "                                                if emote_url and emote_filename:\n",
        "                                                    emote_filepath = os.path.join(dms_dir, \"media\", \"emotes\", f\"{emote_id}-{emote_filename}.webp\")\n",
        "                                                    if not os.path.exists(emote_filepath):\n",
        "                                                        try:\n",
        "                                                            response = requests.get(emote_url)\n",
        "                                                            if response.status_code == 200:\n",
        "                                                                os.makedirs(os.path.dirname(emote_filepath), exist_ok=True)\n",
        "                                                                with open(emote_filepath, \"wb\") as f:\n",
        "                                                                    f.write(response.content)\n",
        "                                                            else:\n",
        "                                                                print(f\"Failed to download custom reaction: {emote_filename} (Status code: {response.status_code})\")\n",
        "                                                        except requests.exceptions.RequestException as e:\n",
        "                                                            print(f\"Error downloading custom reaction: {emote_filename}\")\n",
        "                                                            print(e)\n",
        "                                                            pass\n",
        "                                                    content += f'<img src=\"{os.path.join(\"media\", \"emotes\", f\"{emote_id}-{emote_filename}.webp\")} alt=\"{emote_filename}\" title=\"{emote_filename}\" class=\"emote\">'\n",
        "\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"mention\":\n",
        "                                        mention_data = sub_node.get(\"data\", {}).get(\"mention\", {})\n",
        "                                        mentioned_id = mention_data.get(\"id\")\n",
        "                                        mentioned_name = mention_data.get(\"name\")\n",
        "                                        display_name = f'@{mentioned_name}'\n",
        "                                        content += display_name\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"link\":\n",
        "                                        link_data = sub_node.get(\"data\", {})\n",
        "                                        href = link_data.get(\"href\", \"\")\n",
        "                                        if href:\n",
        "                                            link_text = sub_node.get(\"nodes\", [{}])[0].get(\"leaves\", [{}])[0].get(\"text\", \"\")\n",
        "                                            content += f'<a href=\"{href}\">{link_text}</a>'\n",
        "\n",
        "                                    elif sub_node.get(\"type\") == \"block-quote-container\":\n",
        "                                        block_quote_lines = sub_node.get(\"nodes\", [])\n",
        "                                        block_quote_content = ''\n",
        "                                        for block_quote_line in block_quote_lines:\n",
        "                                            if block_quote_line.get(\"type\") == \"block-quote-line\":\n",
        "                                                block_quote_line_nodes = block_quote_line.get(\"nodes\", [])\n",
        "                                                for block_quote_line_node in block_quote_line_nodes:\n",
        "                                                    leaves = block_quote_line_node.get(\"leaves\", [])\n",
        "                                                    for leaf in leaves:\n",
        "                                                        leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                        marks = leaf.get(\"marks\", [])\n",
        "                                                        for mark in marks:\n",
        "                                                            mark_type = mark.get(\"type\")\n",
        "                                                            if mark_type == \"inline-code-v2\":\n",
        "                                                                leaf_text = f'<code>{leaf_text}</code>'\n",
        "                                                            elif mark_type == \"bold\":\n",
        "                                                                leaf_text = f'<strong>{leaf_text}</strong>'\n",
        "                                                            elif mark_type == \"italic\":\n",
        "                                                                leaf_text = f'<em>{leaf_text}</em>'\n",
        "                                                            elif mark_type == \"underline\":\n",
        "                                                                leaf_text = f'<u>{leaf_text}</u>'\n",
        "                                                            elif mark_type == \"strikethrough\":\n",
        "                                                                leaf_text = f'<del>{leaf_text}</del>'\n",
        "                                                            elif mark_type == \"spoiler\":\n",
        "                                                                leaf_text = f'<span style=\"background-color: #000; color: #fff;\">{leaf_text}</span>'\n",
        "                                                        block_quote_content += leaf_text\n",
        "                                                block_quote_content += '<br>'\n",
        "                                        content += f'<div style=\"background-color: #f0f0f0; border-left: 4px solid #F5C400; padding: 10px;\">{block_quote_content}</div>'\n",
        "\n",
        "\n",
        "                                    else:\n",
        "                                        leaves = sub_node.get(\"leaves\", [])\n",
        "                                        for leaf in leaves:\n",
        "                                            leaf_text = leaf.get(\"text\", \"\")\n",
        "                                            marks = leaf.get(\"marks\", [])\n",
        "                                            for mark in marks:\n",
        "                                                mark_type = mark.get(\"type\")\n",
        "                                                if mark_type == \"inline-code-v2\":\n",
        "                                                    leaf_text = f'<code>{leaf_text}</code>'\n",
        "                                                elif mark_type == \"bold\":\n",
        "                                                    leaf_text = f'<strong>{leaf_text}</strong>'\n",
        "                                                elif mark_type == \"italic\":\n",
        "                                                    leaf_text = f'<em>{leaf_text}</em>'\n",
        "                                                elif mark_type == \"underline\":\n",
        "                                                    leaf_text = f'<u>{leaf_text}</u>'\n",
        "                                                elif mark_type == \"strikethrough\":\n",
        "                                                    leaf_text = f'<del>{leaf_text}</del>'\n",
        "                                            content += leaf_text\n",
        "                                content += '<br>'\n",
        "\n",
        "                            elif node.get(\"type\") == \"block-quote-container\":\n",
        "                                block_quote_lines = node.get(\"nodes\", [])\n",
        "                                block_quote_content = ''\n",
        "                                for block_quote_line in block_quote_lines:\n",
        "                                    if block_quote_line.get(\"type\") == \"block-quote-line\":\n",
        "                                        block_quote_line_nodes = block_quote_line.get(\"nodes\", [])\n",
        "                                        for block_quote_line_node in block_quote_line_nodes:\n",
        "                                            leaves = block_quote_line_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                block_quote_content += '<br>' + leaf_text.lstrip('>').strip()\n",
        "                                content += f'<blockquote style=\"border-left: 4px solid #F5C400; padding: 4px;\">{block_quote_content.strip()}</blockquote>'\n",
        "\n",
        "                            elif node.get(\"type\") == \"unordered-list\":\n",
        "                                list_items = node.get(\"nodes\", [])\n",
        "                                unordered_list_content = ''\n",
        "                                for list_item in list_items:\n",
        "                                    if list_item.get(\"type\") == \"list-item\":\n",
        "                                        list_item_nodes = list_item.get(\"nodes\", [])\n",
        "                                        list_item_content = ''\n",
        "                                        for list_item_node in list_item_nodes:\n",
        "                                            leaves = list_item_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                list_item_content += leaf_text\n",
        "                                        unordered_list_content += f'<li>{list_item_content}</li>'\n",
        "                                content += f'<ul>{unordered_list_content}</ul>'\n",
        "                            elif node.get(\"type\") == \"ordered-list\":\n",
        "                                list_items = node.get(\"nodes\", [])\n",
        "                                ordered_list_content = ''\n",
        "                                for list_item in list_items:\n",
        "                                    if list_item.get(\"type\") == \"list-item\":\n",
        "                                        list_item_nodes = list_item.get(\"nodes\", [])\n",
        "                                        list_item_content = ''\n",
        "                                        for list_item_node in list_item_nodes:\n",
        "                                            leaves = list_item_node.get(\"leaves\", [])\n",
        "                                            for leaf in leaves:\n",
        "                                                leaf_text = leaf.get(\"text\", \"\")\n",
        "                                                list_item_content += leaf_text\n",
        "                                        ordered_list_content += f'<li>{list_item_content}</li>'\n",
        "                                    elif list_item.get(\"type\") == \"ordered-list\":\n",
        "                                        nested_ordered_list_content = ''\n",
        "                                        nested_list_items = list_item.get(\"nodes\", [])\n",
        "                                        for nested_list_item in nested_list_items:\n",
        "                                            if nested_list_item.get(\"type\") == \"list-item\":\n",
        "                                                nested_list_item_nodes = nested_list_item.get(\"nodes\", [])\n",
        "                                                nested_list_item_content = ''\n",
        "                                                for nested_list_item_node in nested_list_item_nodes:\n",
        "                                                    nested_leaves = nested_list_item_node.get(\"leaves\", [])\n",
        "                                                    for nested_leaf in nested_leaves:\n",
        "                                                        nested_leaf_text = nested_leaf.get(\"text\", \"\")\n",
        "                                                        nested_list_item_content += nested_leaf_text\n",
        "                                                nested_ordered_list_content += f'<li>{nested_list_item_content}</li>'\n",
        "                                        ordered_list_content += f'<ol>{nested_ordered_list_content}</ol>'\n",
        "                                content += f'<ol>{ordered_list_content}</ol>'\n",
        "\n",
        "\n",
        "            formatted_message = MESSAGE_TEMPLATE.format(\n",
        "                avatar_url=avatar_url,\n",
        "                created_by=created_by,\n",
        "                created_at=created_at,\n",
        "                content=content\n",
        "            )\n",
        "            formatted_messages.append(formatted_message)\n",
        "\n",
        "        # Wait for all download tasks to complete\n",
        "        for future in as_completed(futures):\n",
        "            future.result()\n",
        "\n",
        "    html = HTML_TEMPLATE.format(\n",
        "        title=dm_data[\"name\"],\n",
        "        topic=\", \".join(member[\"name\"] for member in dm_data[\"users\"]),\n",
        "        messages='\\n'.join(formatted_messages),\n",
        "        CSS=CSS\n",
        "    )\n",
        "\n",
        "    return html\n",
        "\n",
        "\n",
        "\n",
        "myself_data = fetch_myself()\n",
        "with open(os.path.join(SAVE_DIRECTORY, \"myself.json\"), \"w\") as f:\n",
        "    f.write(json.dumps(myself_data, separators=(',', ':')))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}myself.json\")\n",
        "USER_ID = myself_data[\"user\"][\"id\"]\n",
        "dms_data = fetch_dms()\n",
        "with open(os.path.join(SAVE_DIRECTORY, \"dms.json\"), \"w\") as f:\n",
        "    f.write(json.dumps(dms_data, separators=(',', ':')))\n",
        "    print(f\"Saved {SAVE_DIRECTORY}dms.json\")\n",
        "\n",
        "\n",
        "for channel in dms_data['channels']:\n",
        "    channel_id = channel['id']\n",
        "    if verbose_dmfetch:\n",
        "        print(f\"found {channel_id}\")\n",
        "    dm_data = fetch_channel(channel_id)\n",
        "    partner = next((user for user in channel['users'] if user['id'] != USER_ID), None)\n",
        "    if channel.get(\"dmType\") == \"Group\":\n",
        "        dm_foldername = f\"Group {channel_id}\"\n",
        "    elif partner:\n",
        "        dm_foldername = f\"{partner['name']} ({partner['id']})\"\n",
        "    else:\n",
        "        dm_foldername = f\"[deleted user] ({channel_id})\"\n",
        "    dm_dir = os.path.join(SAVE_DIRECTORY, \"Direct Messages\", sanitize_filename(dm_foldername))\n",
        "\n",
        "    # Check if the directory already exists\n",
        "    if os.path.exists(dm_dir):\n",
        "        print(f\"Skipping existing directory: {dm_dir}\")\n",
        "        continue  # Skip to the next channel\n",
        "\n",
        "    os.makedirs(dm_dir, exist_ok=True)\n",
        "    with open(os.path.join(dm_dir, f\"{channel_id}_info.json\"), 'w') as f:\n",
        "        json.dump(dm_data, f, separators=(',', ':'))\n",
        "        if verbose_dmfetch:\n",
        "            print(f\"Saving channel {channel_id} for user {partner['name'] if partner else 'Unknown'}\")\n",
        "    members_dict = {}\n",
        "    for member in dm_data['users']:\n",
        "        members_dict[member['name']] = member\n",
        "\n",
        "    url = f\"https://www.guilded.gg/api/channels/{channel_id}/messages\"\n",
        "    beforeDate = None # Reset beforeDate for each channel\n",
        "    messages = [] # Reset messages for each channel\n",
        "    mesc = 0 # Reset mesc for each channel\n",
        "    while True:\n",
        "        params = {\"maxItems\": 100, \"beforeDate\": beforeDate}\n",
        "        response = requests.get(url, params=params, cookies=cookies)\n",
        "        dm_mes = unshid_cdn(response.json())\n",
        "        #response = fetch(f\"api/channels/{channel_id}/messages\", params={\"maxItems\": 100, \"beforeDate\": beforeDate})\n",
        "        messages.extend(dm_mes[\"messages\"])\n",
        "        mesc += len(dm_mes[\"messages\"])\n",
        "        if verbose_dmfetch:\n",
        "            print(f\"Collected {mesc} messages\")\n",
        "        if len(dm_mes[\"messages\"]) < 100:\n",
        "            break\n",
        "        beforeDate = dm_mes[\"messages\"][-1][\"createdAt\"]\n",
        "    messages.reverse()\n",
        "    # Save messages JSON\n",
        "    with open(os.path.join(dm_dir, f\"{channel_id}_messages.json\"), 'w') as f:\n",
        "        json.dump(messages, f, separators=(',', ':'))\n",
        "    if verbose_dmfetch:\n",
        "        print(f\"Found {len(messages)} messages\")\n",
        "\n",
        "    # Generate and save HTML\n",
        "    if len(messages) > messages_per_page:\n",
        "        for i in range(0, len(messages), messages_per_page):\n",
        "            html = generate_html(messages[i:i+messages_per_page], dm_data, members_dict, dm_dir)\n",
        "            html_filename = f\"DMs with {partner['name']} - Page {i//messages_per_page + 1}.html\".replace('/', '-')\n",
        "            if channel.get(\"dmType\") == \"Group\":\n",
        "                html_filename = f\"Group {channel_id} - Page {i//messages_per_page + 1}.html\".replace('/', '-')\n",
        "            with open(os.path.join(dm_dir, html_filename), 'w') as f:\n",
        "                f.write(html)\n",
        "            if verbose_dmfetch:\n",
        "                print(f\"Saved {dm_dir}/{html_filename}\")\n",
        "    else:\n",
        "        html = generate_html(messages, dm_data, members_dict, dm_dir)\n",
        "        html_filename = f\"DMs with {partner['name']}.html\".replace('/', '-')\n",
        "        if channel.get(\"dmType\") == \"Group\":\n",
        "            html_filename = f\"Group {channel_id}.html\".replace('/', '-')\n",
        "        with open(os.path.join(dm_dir, html_filename), 'w') as f:\n",
        "            f.write(html)\n",
        "\n",
        "    print(f\"Saved HTML to {dm_dir}/{html_filename}\")\n",
        "\n",
        "print(\"Finished processing all DMs.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "x9cSXn8BYvLD"
      },
      "outputs": [],
      "source": [
        "# @title (Optional) Get the size of the directory SAVE_DIRECTORY + \"Direct Messages\"\n",
        "\n",
        "import os\n",
        "\n",
        "todl = SAVE_DIRECTORY + \"Direct Messages\"\n",
        "\n",
        "size = 0\n",
        "for root, dirs, files in os.walk(todl):\n",
        "    for file in files:\n",
        "        size += os.path.getsize(os.path.join(root, file))\n",
        "\n",
        "mb = size / (1024 * 1024)\n",
        "gb = size / (1024 * 1024 * 1024)\n",
        "\n",
        "print(f\"Size of directory: {mb:.2f} MB / {gb:.2f} GB\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "8D0ciWPvOoKu"
      },
      "outputs": [],
      "source": [
        "# @title Download and Save DMs (dont ask why it's different here)\n",
        "save_type = \"7z\" # @param [\"7z\", \"targz\", \"zip\"]\n",
        "saving_to = \"DirectDownLoad\" # @param [\"DirectDownLoad\", \"GoogleDrive\"]\n",
        "from google.colab import drive\n",
        "from google.colab import files\n",
        "\n",
        "todl = SAVE_DIRECTORY + \"Direct Messages\"\n",
        "if save_type == \"7z\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}Direct_Messages.7z\"\n",
        "    !7z a -mx {outfile} \"{todl}\" -mx0\n",
        "if save_type == \"targz\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}Direct_Messages.tar.gz\"\n",
        "    !tar -czvf {outfile} \"{todl}\"\n",
        "if save_type == \"zip\":\n",
        "    outfile = f\"{SAVE_DIRECTORY}Direct_Messages.zip\"\n",
        "    !zip -r {outfile} \"{todl}\"\n",
        "\n",
        "if saving_to == \"DirectDownLoad\":\n",
        "    files.download(outfile)\n",
        "if saving_to == \"GoogleDrive\":\n",
        "    drive.mount('/content/drive')\n",
        "    !cp \"{outfile}\" /content/drive/MyDrive\n",
        "\n",
        "#if save_to_gdrive:\n",
        "    #shutil.copy(SERVER_NAME + '.zip', '/content/drive/MyDrive/' + SERVER_NAME + '.zip')\n",
        "    #print(\"Copied zip to Google Drive.\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "OP9gMhMuRhQH"
      },
      "outputs": [],
      "source": [
        "# @title Profile Info Nabber (Self and Friends)\n",
        "me = fetch_myself()\n",
        "friends = me.get(\"friends\")\n",
        "friend_ids = [friend[\"friendUserId\"] for friend in friends]\n",
        "myname = me.get('user', {}).get('name')\n",
        "def fetch_userp(user_id):\n",
        "    data = fetch(f\"users/{user_id}\")\n",
        "    return data[\"user\"]\n",
        "\n",
        "profiledir = os.path.join(SAVE_DIRECTORY, \"Profiles\")\n",
        "data = fetch_userp(me.get('user', {}).get('id'))\n",
        "os.makedirs(profiledir, exist_ok=True)\n",
        "with open(os.path.join(profiledir, f\"ProfileMyself-{myname}.json\"), \"w\") as f:\n",
        "        f.write(json.dumps(data, separators=(',', ':')))\n",
        "        print(f\"Saved {profiledir}/ProfileMyself-{myname}.json\")\n",
        "for member in dm_data[\"users\"]:\n",
        "    data = fetch_user(member[\"id\"])\n",
        "    with open(os.path.join(profiledir, f\"{member['id']}-{member['name']}.json\"), \"w\") as f:\n",
        "        f.write(json.dumps(data, separators=(',', ':')))\n",
        "        print(f\"Saved {profiledir}/{member['id']}-{member['name']}.json\")\n",
        "for friend_id in friend_ids:\n",
        "    data = fetch_user(friend_id)\n",
        "    with open(os.path.join(profiledir, f\"{friend_id}-{data['name']}.json\"), \"w\") as f:\n",
        "        f.write(json.dumps(data, separators=(',', ':')))\n",
        "        print(f\"Saved {profiledir}/{friend_id}-{data['name']}.json\")\n",
        "for member in members_dict:\n",
        "    data = fetch_user(members_dict[member][\"id\"])\n",
        "    with open(os.path.join(profiledir, f\"{members_dict[member]['id']}-{members_dict[member]['name']}.json\"), \"w\") as f:\n",
        "        f.write(json.dumps(data, separators=(',', ':')))\n",
        "        print(f\"Saved {profiledir}/{members_dict[member]['id']}-{members_dict[member]['name']}\")\n",
        "\n",
        "# zip and download profiledir\n",
        "outfile = f\"{SAVE_DIRECTORY}Profiles.7z\"\n",
        "!7z a {outfile} {profiledir}\n",
        "files.download(outfile)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "0-HEnz9fOAuo"
      },
      "outputs": [],
      "source": [
        "# @title (Debug) Preview HTML (wont show local files)\n",
        "# @markdown You can test how messages look. But you have to put the link manually, I changed how folders work too much\n",
        "from IPython.display import HTML\n",
        "\n",
        "directory = \"\" # @param {type:\"string\"}\n",
        "\n",
        "with open(directory, \"r\") as f:\n",
        "    html_content = f.read()\n",
        "display(HTML(html_content))\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPOKRCPnx1JohgH7YM8x3gC",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}