feat: pre-messages / next version of download on demand #7371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

link2xt merged 39 commits into main from pre-messages

Jan 8, 2026

+2,835 −1,385

deltachat-ffi/deltachat.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -485,12 +485,15 @@ char* dc_get_blobdir (const dc_context_t* context); @@
      *                    0=use IMAP IDLE if the server supports it.
      *                    This is a developer option used for testing polling used as an IDLE fallback.
      * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
-     *                    For larger messages, only the header is downloaded and a placeholder is shown.
-     *                    These messages can be downloaded fully using dc_download_full_msg() later.
-     *                    The limit is compared against raw message sizes, including headers.
-     *                    The actually used limit may be corrected
-     *                    to not mess up with non-delivery-reports or read-receipts.
-     *                    0=no limit (default).
+     *                    For messages with large attachments, two messages are sent:
+     *                    a Pre-Message containing metadata and text and a Post-Message additionally
+     *                    containing the attachment. NB: Some "extra" metadata like avatars and gossiped
+     *                    encryption keys is stripped from post-messages to save traffic.
+     *                    Pre-Messages are shown as placeholder messages. They can be downloaded fully
+     *                    using dc_download_full_msg() later. Post-Messages are automatically
+     *                    downloaded if they are smaller than the download_limit. Other messages are
+     *                    always auto-downloaded.
+     *                    0 = no limit (default).
      *                    Changes affect future messages only.
      * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
      *                    This is an experimental option not compatible to other MUAs
@@ Expand Down Expand Up / @@ -4312,6 +4315,7 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); @@
     /**
      * Get the size of the file. Returns the size of the file associated with a
      * message, if applicable.
+     * If message is a pre-message, then this returns the size of the file to be downloaded.
      *
      * Typically, this is used to show the size of document files, e.g. a PDF.
      *
@@ Expand Down Expand Up / @@ -7274,22 +7278,9 @@ void dc_event_unref(dc_event_t* event); @@
     /// `%1$s` will be replaced by the percentage used
     #define DC_STR_QUOTA_EXCEEDING_MSG_BODY   98
-    /// "%1$s message"
-    ///
-    /// Used as the message body when a message
-    /// was not yet downloaded completely
-    /// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE).
-    ///
-    /// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB").
+    /// @deprecated Deprecated 2025-11-12, this string is no longer needed.
     #define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY  99
-    /// "Download maximum available until %1$s"
-    ///
-    /// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY.
-    ///
-    /// `%1$s` will be replaced by human-readable date and time.
-    #define DC_STR_DOWNLOAD_AVAILABILITY      100
     /// "Multi Device Synchronization"
     ///
     /// Used in subjects of outgoing sync messages.
@@ Expand Down @@

deltachat-jsonrpc/src/api/types/message.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -92,6 +92,9 @@ pub struct MessageObject { @@
         file: Option<String>,
         file_mime: Option<String>,
+        /// The size of the file in bytes, if applicable.
+        /// If message is a pre-message, then this is the size of the file to be downloaded.
         file_bytes: u64,
         file_name: Option<String>,
@@ Expand Down @@

deltachat-rpc-client/tests/test_chatlist_events.py

-Original file line number
+Diff line change
@@ -1,7 +1,5 @@
     from __future__ import annotations
-    import base64
-    import os
     from typing import TYPE_CHECKING
     from deltachat_rpc_client import Account, EventType, const
@@ Expand Down Expand Up / @@ -129,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None: @@
         msg.get_snapshot().chat.accept()
         bob.get_chat_by_id(chat_id).send_message(
             "Hello World, this message is bigger than 5 bytes",
-            html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
+            file="../test-data/image/screenshot.jpg",
         )
         message = alice.wait_for_incoming_msg()
@@ Expand Down @@

deltachat-rpc-client/tests/test_something.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -5,12 +5,11 @@
  
    import os

    import socket

    import subprocess

    import time

    from unittest.mock import MagicMock

    import pytest

    from deltachat_rpc_client import Contact, EventType, Message, events

    from deltachat_rpc_client import EventType, events

    from deltachat_rpc_client.const import DownloadState, MessageState

    from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS

    from deltachat_rpc_client.rpc import JsonRpcError

    @@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None:
  
        alice_contact_bob = alice.create_contact(bob, "Bob")

        alice_chat_bob = alice_contact_bob.create_chat()

        bob.set_config("fail_on_receiving_full_msg", "1")

        bob.set_config("simulate_receive_imf_error", "1")

        alice_chat_bob.send_text("Hello!")

        event = bob.wait_for_event(EventType.MSGS_CHANGED)

        assert event.chat_id == bob.get_device_chat().id

    @@ -343,18 +342,17 @@ def test_receive_imf_failure(acfactory) -> None:
  
        version = bob.get_info()["deltachat_core_version"]

        assert (

            snapshot.text == "❌ Failed to receive a message:"

            " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`."

            " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`."

            f" Core version {version}."

            " Please report this bug to delta@merlinux.eu or https://support.delta.chat/."

        )

        # The failed message doesn't break the IMAP loop.

        bob.set_config("fail_on_receiving_full_msg", "0")

        bob.set_config("simulate_receive_imf_error", "0")

        alice_chat_bob.send_text("Hello again!")

        message = bob.wait_for_incoming_msg()

        snapshot = message.get_snapshot()

        assert snapshot.text == "Hello again!"

        assert snapshot.download_state == DownloadState.DONE

        assert snapshot.error is None

    @@ -687,60 +685,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
  
        assert snapshot.show_padlock

    def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):

        """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded

        messages are received out of order".

        If the Inbox contains X small messages followed by Y large messages followed by Z small

        messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages.

        This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced

        with online test as follows:

        - Bob enables download limit and goes offline.

        - Alice sends a large message to Bob and reacts to this message with a thumbs-up.

        - Bob goes online

        - Bob first processes a reaction message and throws it away because there is no corresponding

          message, then processes a partially downloaded message.

        - As a result, Bob does not see a reaction

        """

        download_limit = 300000

        ac1, ac2 = acfactory.get_online_accounts(2)

        ac1_addr = ac1.get_config("addr")

        chat = ac1.create_chat(ac2)

        ac2.set_config("download_limit", str(download_limit))

        ac2.stop_io()

        logging.info("sending small+large messages from ac1 to ac2")

        msgs = []

        msgs.append(chat.send_text("hi"))

        path = tmp_path / "large"

        path.write_bytes(os.urandom(download_limit + 1))

        msgs.append(chat.send_file(str(path)))

        for m in msgs:

            m.wait_until_delivered()

        logging.info("sending a reaction to the large message from ac1 to ac2")

        # TODO: Find the reason of an occasional message reordering on the server (so that the reaction

        # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction

        # have a later INTERNALDATE.

        time.sleep(1.1)

        react_str = "\N{THUMBS UP SIGN}"

        msgs.append(msgs[-1].send_reaction(react_str))

        msgs[-1].wait_until_delivered()

        ac2.start_io()

        logging.info("wait for ac2 to receive a reaction")

        msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)

        assert msg2.get_sender_contact().get_snapshot().address == ac1_addr

        assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE

        reactions = msg2.get_reactions()

        contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]

        assert len(contacts) == 1

        assert contacts[0].get_snapshot().address == ac1_addr

        assert list(reactions.reactions_by_contact.values())[0] == [react_str]

    @pytest.mark.parametrize("n_accounts", [3, 2])

    def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):

        download_limit = 300000

    @@ -767,14 +711,159 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
  
        path = tmp_path / "large"

        path.write_bytes(os.urandom(download_limit + 1))

        n_done = 0

        for i in range(10):

            logging.info("Sending message %s", i)

            alice_group.send_file(str(path))

            snapshot = bob.wait_for_incoming_msg().get_snapshot()

            assert snapshot.download_state == DownloadState.AVAILABLE

            if snapshot.download_state == DownloadState.DONE:

                n_done += 1

                # Work around lost and reordered pre-messages.

                assert n_done <= 1

            else:

                assert snapshot.download_state == DownloadState.AVAILABLE

            assert snapshot.chat == bob_group

    def test_download_small_msg_first(acfactory, tmp_path):

        download_limit = 70000

        alice, bob0 = acfactory.get_online_accounts(2)

        bob1 = bob0.clone()

        bob1.set_config("download_limit", str(download_limit))

        chat = alice.create_chat(bob0)

        path = tmp_path / "large_enough"

        path.write_bytes(os.urandom(download_limit + 1))

        # Less than 140K, so sent w/o a pre-message.

        chat.send_file(str(path))

        chat.send_text("hi")

        bob0.create_chat(alice)

        assert bob0.wait_for_incoming_msg().get_snapshot().text == ""

        assert bob0.wait_for_incoming_msg().get_snapshot().text == "hi"

        bob1.start_io()

        bob1.create_chat(alice)

        assert bob1.wait_for_incoming_msg().get_snapshot().text == "hi"

        assert bob1.wait_for_incoming_msg().get_snapshot().text == ""

    @pytest.mark.parametrize("delete_chat", [False, True])

    def test_delete_available_msg(acfactory, tmp_path, direct_imap, delete_chat):

        """

        Tests `DownloadState.AVAILABLE` message deletion on the receiver side.

        Also tests pre- and post-message deletion on the sender side.

        """

        # Min. UI setting as of v2.35

        download_limit = 163840

        alice, bob = acfactory.get_online_accounts(2)

        bob.set_config("download_limit", str(download_limit))

        # Avoid immediate deletion from the server

        alice.set_config("bcc_self", "1")

        bob.set_config("bcc_self", "1")

        chat_alice = alice.create_chat(bob)

        path = tmp_path / "large"

        path.write_bytes(os.urandom(download_limit + 1))

        msg_alice = chat_alice.send_file(str(path))

        msg_bob = bob.wait_for_incoming_msg()

        msg_bob_snapshot = msg_bob.get_snapshot()

        assert msg_bob_snapshot.download_state == DownloadState.AVAILABLE

        chat_bob = bob.get_chat_by_id(msg_bob_snapshot.chat_id)

        # Avoid DeleteMessages sync message

        bob.set_config("bcc_self", "0")

        if delete_chat:

            chat_bob.delete()

        else:

            bob.delete_messages([msg_bob])

        alice.wait_for_event(EventType.SMTP_MESSAGE_SENT)

        alice.wait_for_event(EventType.SMTP_MESSAGE_SENT)

        alice.set_config("bcc_self", "0")

        if delete_chat:

            chat_alice.delete()

        else:

            alice.delete_messages([msg_alice])

        for acc in [bob, alice]:

            if not delete_chat:

                acc.wait_for_event(EventType.MSG_DELETED)

            acc_direct_imap = direct_imap(acc)

            # Messages may be deleted separately

            while True:

                acc.wait_for_event(EventType.IMAP_MESSAGE_DELETED)

                while True:

                    event = acc.wait_for_event()

                    if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:

                        break

                if len(acc_direct_imap.get_all_messages()) == 0:

                    break

    def test_delete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):

        alice, bob = acfactory.get_online_accounts(2)

        # Avoid immediate deletion from the server

        bob.set_config("bcc_self", "1")

        chat_alice = alice.create_chat(bob)

        path = tmp_path / "large"

        # Big enough to be sent with a pre-message

        path.write_bytes(os.urandom(300000))

        chat_alice.send_file(str(path))

        msg = bob.wait_for_incoming_msg()

        msg_snapshot = msg.get_snapshot()

        assert msg_snapshot.download_state == DownloadState.AVAILABLE

        msgs_changed_event = bob.wait_for_msgs_changed_event()

        assert msgs_changed_event.msg_id == msg.id

        msg_snapshot = msg.get_snapshot()

        assert msg_snapshot.download_state == DownloadState.DONE

        bob_direct_imap = direct_imap(bob)

        assert len(bob_direct_imap.get_all_messages()) == 2

        # Avoid DeleteMessages sync message

        bob.set_config("bcc_self", "0")

        bob.delete_messages([msg])

        bob.wait_for_event(EventType.MSG_DELETED)

        # Messages may be deleted separately

        while True:

            bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED)

            while True:

                event = bob.wait_for_event()

                if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:

                    break

            if len(bob_direct_imap.get_all_messages()) == 0:

                break

    def test_imap_autodelete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):

        alice, bob = acfactory.get_online_accounts(2)

        chat_alice = alice.create_chat(bob)

        path = tmp_path / "large"

        # Big enough to be sent with a pre-message

        path.write_bytes(os.urandom(300000))

        chat_alice.send_file(str(path))

        msg = bob.wait_for_incoming_msg()

        msg_snapshot = msg.get_snapshot()

        assert msg_snapshot.download_state == DownloadState.AVAILABLE

        msgs_changed_event = bob.wait_for_msgs_changed_event()

        assert msgs_changed_event.msg_id == msg.id

        msg_snapshot = msg.get_snapshot()

        assert msg_snapshot.download_state == DownloadState.DONE

        bob_direct_imap = direct_imap(bob)

        # Messages may be deleted separately

        while True:

            if len(bob_direct_imap.get_all_messages()) == 0:

                break

            bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED)

            while True:

                event = bob.wait_for_event()

                if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:

                    break

    def test_markseen_contact_request(acfactory):

        """

        Test that seen status is synchronized for contact request messages

    @@ -1152,3 +1241,23 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log):
  
        assert chat.num_contacts() == 2

        assert msg.get_snapshot().chat.num_contacts() == 2

    def test_large_message(acfactory) -> None:

        """

        Test sending large message without download limit set,

        so it is sent with pre-message but downloaded without user interaction.

        """

        alice, bob = acfactory.get_online_accounts(2)

        alice_chat_bob = alice.create_chat(bob)

        alice_chat_bob.send_message(

            "Hello World, this message is bigger than 5 bytes",

            file="../test-data/image/screenshot.jpg",

        )

        msg = bob.wait_for_incoming_msg()

        msgs_changed_event = bob.wait_for_msgs_changed_event()

        assert msg.id == msgs_changed_event.msg_id

        snapshot = msg.get_snapshot()

        assert snapshot.text == "Hello World, this message is bigger than 5 bytes"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feat: pre-messages / next version of download on demand #7371

Diff view

Diff view

There are no files selected for viewing

Uh oh!