From f41b1734fe2c04119e4ef52414ce1526bee42cab Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Wed, 29 Oct 2025 21:50:58 +0100 Subject: [PATCH 01/39] feat: message previews - Remove partial downloads (remove creation of the stub messages) (#7373) - Remove "Download maximum available until" and remove stock string `DC_STR_DOWNLOAD_AVAILABILITY` (#7369) - Send pre-message on messages with large attachments (#7410) - Pre messages can now get read receipts (#7433) Co-authored-by: Hocuri Co-authored-by: iequidoo --- deltachat-ffi/deltachat.h | 31 +- deltachat-jsonrpc/src/api/types/message.rs | 3 + .../tests/test_chatlist_events.py | 4 +- deltachat-rpc-client/tests/test_something.py | 100 +--- python/tests/test_1_online.py | 33 -- src/calls/calls_tests.rs | 64 +-- src/chat.rs | 142 ++++- src/chat/chat_tests.rs | 4 +- src/config.rs | 4 +- src/context.rs | 7 - src/context/context_tests.rs | 1 + src/download.rs | 487 ++++++---------- src/download/pre_msg_metadata.rs | 248 +++++++++ src/headerdef.rs | 18 + src/imap.rs | 127 +++-- src/imap/session.rs | 2 + src/internals_for_benches.rs | 2 +- src/message.rs | 99 +++- src/message/message_tests.rs | 106 ---- src/mimefactory.rs | 103 +++- src/mimefactory/mimefactory_tests.rs | 8 +- src/mimeparser.rs | 111 ++-- src/mimeparser/mimeparser_tests.rs | 129 ++--- src/param.rs | 30 + src/reaction.rs | 70 +-- src/receive_imf.rs | 274 +++++---- src/receive_imf/receive_imf_tests.rs | 273 +-------- src/scheduler.rs | 41 +- src/sql/migrations.rs | 24 + src/stock_str.rs | 49 +- src/stock_str/stock_str_tests.rs | 8 - src/test_utils.rs | 44 +- src/tests.rs | 1 + src/tests/pre_messages.rs | 6 + src/tests/pre_messages/additional_text.rs | 40 ++ src/tests/pre_messages/forward_and_save.rs | 122 ++++ src/tests/pre_messages/legacy.rs | 61 ++ src/tests/pre_messages/receiving.rs | 522 ++++++++++++++++++ src/tests/pre_messages/sending.rs | 337 +++++++++++ src/tests/pre_messages/util.rs | 65 +++ src/webxdc/webxdc_tests.rs | 66 +-- 41 files changed, 2413 insertions(+), 1453 deletions(-) create mode 100644 src/download/pre_msg_metadata.rs create mode 100644 src/tests/pre_messages.rs create mode 100644 src/tests/pre_messages/additional_text.rs create mode 100644 src/tests/pre_messages/forward_and_save.rs create mode 100644 src/tests/pre_messages/legacy.rs create mode 100644 src/tests/pre_messages/receiving.rs create mode 100644 src/tests/pre_messages/sending.rs create mode 100644 src/tests/pre_messages/util.rs diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 6d47d4191a..18ac329092 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -485,11 +485,11 @@ char* dc_get_blobdir (const dc_context_t* context); * 0=use IMAP IDLE if the server supports it. * This is a developer option used for testing polling used as an IDLE fallback. * - `download_limit` = Messages up to this number of bytes are downloaded automatically. - * For larger messages, only the header is downloaded and a placeholder is shown. + * For messages with large attachments, two messages are sent: + * a Pre-Message containing metadata and a Post-Message containing the attachment. + * Pre-Messages are always downloaded and show a placeholder message. * These messages can be downloaded fully using dc_download_full_msg() later. - * The limit is compared against raw message sizes, including headers. - * The actually used limit may be corrected - * to not mess up with non-delivery-reports or read-receipts. + * Post-Messages are automatically downloaded if they are smaller than the download_limit. * 0=no limit (default). * Changes affect future messages only. * - `protect_autocrypt` = Enable Header Protection for Autocrypt header. @@ -4311,7 +4311,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); /** * Get the size of the file. Returns the size of the file associated with a - * message, if applicable. + * message, if applicable. + * If message is a pre-message, then this returns size of the to be downloaded file. * * Typically, this is used to show the size of document files, e.g. a PDF. * @@ -7274,22 +7275,9 @@ void dc_event_unref(dc_event_t* event); /// `%1$s` will be replaced by the percentage used #define DC_STR_QUOTA_EXCEEDING_MSG_BODY 98 -/// "%1$s message" -/// -/// Used as the message body when a message -/// was not yet downloaded completely -/// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE). -/// -/// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB"). +/// @deprecated Deprecated 2025-11-12, this string is no longer needed. #define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY 99 -/// "Download maximum available until %1$s" -/// -/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY. -/// -/// `%1$s` will be replaced by human-readable date and time. -#define DC_STR_DOWNLOAD_AVAILABILITY 100 - /// "Multi Device Synchronization" /// /// Used in subjects of outgoing sync messages. @@ -7781,6 +7769,11 @@ void dc_event_unref(dc_event_t* event); /// Used as the first info messages in newly created classic email threads. #define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230 +/// "Contact" +/// +/// Used in summaries. +#define DC_STR_CONTACT 231 + /** * @} */ diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 567e3ffdc9..27ba8dca0e 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -92,6 +92,9 @@ pub struct MessageObject { file: Option, file_mime: Option, + + /// The size of the file in bytes, if applicable. + /// If message is a pre-message, then this is the size of the to be downloaded file. file_bytes: u64, file_name: Option, diff --git a/deltachat-rpc-client/tests/test_chatlist_events.py b/deltachat-rpc-client/tests/test_chatlist_events.py index 05bec0795f..f13769e071 100644 --- a/deltachat-rpc-client/tests/test_chatlist_events.py +++ b/deltachat-rpc-client/tests/test_chatlist_events.py @@ -1,7 +1,5 @@ from __future__ import annotations -import base64 -import os from typing import TYPE_CHECKING from deltachat_rpc_client import Account, EventType, const @@ -129,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None: msg.get_snapshot().chat.accept() bob.get_chat_by_id(chat_id).send_message( "Hello World, this message is bigger than 5 bytes", - html=base64.b64encode(os.urandom(300000)).decode("utf-8"), + file="../test-data/image/screenshot.jpg", ) message = alice.wait_for_incoming_msg() diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index b6337e01c2..eda0bc1166 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -5,13 +5,12 @@ import os import socket import subprocess -import time from unittest.mock import MagicMock import pytest -from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client import EventType, events +from deltachat_rpc_client.const import MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError @@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None: alice_contact_bob = alice.create_contact(bob, "Bob") alice_chat_bob = alice_contact_bob.create_chat() - bob.set_config("fail_on_receiving_full_msg", "1") + bob.set_config("simulate_receive_imf_error", "1") alice_chat_bob.send_text("Hello!") event = bob.wait_for_event(EventType.MSGS_CHANGED) assert event.chat_id == bob.get_device_chat().id @@ -343,18 +342,17 @@ def test_receive_imf_failure(acfactory) -> None: version = bob.get_info()["deltachat_core_version"] assert ( snapshot.text == "❌ Failed to receive a message:" - " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`." + " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`." f" Core version {version}." " Please report this bug to delta@merlinux.eu or https://support.delta.chat/." ) # The failed message doesn't break the IMAP loop. - bob.set_config("fail_on_receiving_full_msg", "0") + bob.set_config("simulate_receive_imf_error", "0") alice_chat_bob.send_text("Hello again!") message = bob.wait_for_incoming_msg() snapshot = message.get_snapshot() assert snapshot.text == "Hello again!" - assert snapshot.download_state == DownloadState.DONE assert snapshot.error is None @@ -687,94 +685,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None: assert snapshot.show_padlock -def test_reaction_to_partially_fetched_msg(acfactory, tmp_path): - """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded - messages are received out of order". - - If the Inbox contains X small messages followed by Y large messages followed by Z small - messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages. - - This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced - with online test as follows: - - Bob enables download limit and goes offline. - - Alice sends a large message to Bob and reacts to this message with a thumbs-up. - - Bob goes online - - Bob first processes a reaction message and throws it away because there is no corresponding - message, then processes a partially downloaded message. - - As a result, Bob does not see a reaction - """ - download_limit = 300000 - ac1, ac2 = acfactory.get_online_accounts(2) - ac1_addr = ac1.get_config("addr") - chat = ac1.create_chat(ac2) - ac2.set_config("download_limit", str(download_limit)) - ac2.stop_io() - - logging.info("sending small+large messages from ac1 to ac2") - msgs = [] - msgs.append(chat.send_text("hi")) - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - msgs.append(chat.send_file(str(path))) - for m in msgs: - m.wait_until_delivered() - - logging.info("sending a reaction to the large message from ac1 to ac2") - # TODO: Find the reason of an occasional message reordering on the server (so that the reaction - # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction - # have a later INTERNALDATE. - time.sleep(1.1) - react_str = "\N{THUMBS UP SIGN}" - msgs.append(msgs[-1].send_reaction(react_str)) - msgs[-1].wait_until_delivered() - - ac2.start_io() - - logging.info("wait for ac2 to receive a reaction") - msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id) - assert msg2.get_sender_contact().get_snapshot().address == ac1_addr - assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE - reactions = msg2.get_reactions() - contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact] - assert len(contacts) == 1 - assert contacts[0].get_snapshot().address == ac1_addr - assert list(reactions.reactions_by_contact.values())[0] == [react_str] - - -@pytest.mark.parametrize("n_accounts", [3, 2]) -def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): - download_limit = 300000 - - alice, *others = acfactory.get_online_accounts(n_accounts) - bob = others[0] - - alice_group = alice.create_group("test group") - for account in others: - chat = account.create_chat(alice) - chat.send_text("Hello Alice!") - assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!" - - contact = alice.create_contact(account) - alice_group.add_contact(contact) - - bob.set_config("download_limit", str(download_limit)) - - alice_group.send_text("hi") - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.text == "hi" - bob_group = snapshot.chat - - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - - for i in range(10): - logging.info("Sending message %s", i) - alice_group.send_file(str(path)) - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.download_state == DownloadState.AVAILABLE - assert snapshot.chat == bob_group - - def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 0bfcca0759..51e760eccc 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1,7 +1,6 @@ import os import queue import sys -import base64 from datetime import datetime, timezone import pytest @@ -221,38 +220,6 @@ def test_webxdc_huge_update(acfactory, data, lp): assert update["payload"] == payload -def test_webxdc_download_on_demand(acfactory, data, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - acfactory.introduce_each_other([ac1, ac2]) - chat = acfactory.get_accepted_chat(ac1, ac2) - - msg1 = Message.new_empty(ac1, "webxdc") - msg1.set_text("message1") - msg1.set_file(data.get_path("webxdc/minimal.xdc")) - msg1 = chat.send_msg(msg1) - assert msg1.is_webxdc() - assert msg1.filename - - msg2 = ac2._evtracker.wait_next_incoming_message() - assert msg2.is_webxdc() - - lp.sec("ac2 sets download limit") - ac2.set_config("download_limit", "100") - assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data") - ac2_update = ac2._evtracker.wait_next_incoming_message() - assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE - assert not msg2.get_status_updates() - - ac2_update.download_full() - ac2._evtracker.get_matching("DC_EVENT_WEBXDC_STATUS_UPDATE") - assert msg2.get_status_updates() - - # Get a event notifying that the message disappeared from the chat. - msgs_changed_event = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert msgs_changed_event.data1 == msg2.chat.id - assert msgs_changed_event.data2 == 0 - - def test_enable_mvbox_move(acfactory, lp): (ac1,) = acfactory.get_online_accounts(1) diff --git a/src/calls/calls_tests.rs b/src/calls/calls_tests.rs index 3f983d8438..23b3947a2e 100644 --- a/src/calls/calls_tests.rs +++ b/src/calls/calls_tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::chat::forward_msgs; use crate::config::Config; use crate::constants::DC_CHAT_ID_TRASH; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; struct CallSetup { @@ -610,65 +610,3 @@ async fn test_end_text_call() -> Result<()> { Ok(()) } - -/// Tests that partially downloaded "call ended" -/// messages are not processed. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_no_partial_calls() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let seen = false; - - // The messages in the test - // have no `Date` on purpose, - // so they are treated as new. - let received_call = receive_imf( - alice, - b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call\n\ - Chat-Webrtc-Room: YWFhYWFhYWFhCg==\n\ - \n\ - Hello, this is a call\n", - seen, - ) - .await? - .unwrap(); - assert_eq!(received_call.msg_ids.len(), 1); - let call_msg = Message::load_from_db(alice, received_call.msg_ids[0]) - .await - .unwrap(); - assert_eq!(call_msg.viewtype, Viewtype::Call); - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - let imf_raw = b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - In-Reply-To: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call-ended\n\ - \n\ - Call ended\n"; - receive_imf_from_inbox( - alice, - "second@example.net", - imf_raw, - seen, - Some(imf_raw.len().try_into().unwrap()), - ) - .await?; - - // The call is still not ended. - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - // Fully downloading the message ends the call. - receive_imf_from_inbox(alice, "second@example.net", imf_raw, seen, None) - .await - .context("Failed to fully download end call message")?; - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Missed); - - Ok(()) -} diff --git a/src/chat.rs b/src/chat.rs index de3433fda9..42552c0f4b 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -12,6 +12,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use chrono::TimeZone; use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line}; +use humansize::{BINARY, format_size}; use mail_builder::mime::MimePart; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; @@ -27,7 +28,9 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::DownloadState; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD, +}; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; use crate::key::self_fingerprint; @@ -35,7 +38,7 @@ use crate::location; use crate::log::{LogExt, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::mimefactory::MimeFactory; +use crate::mimefactory::{MimeFactory, RenderedEmail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::receive_imf::ReceivedMsg; @@ -2736,6 +2739,57 @@ async fn prepare_send_msg( Ok(row_ids) } +/// Renders the Message or splits it into Post-Message and Pre-Message. +/// +/// Pre-Message is a small message with metadata which announces a larger Post-Message. +/// Post-Messages are not downloaded in the background. +/// +/// If pre-message is not nessesary this returns a normal message instead. +async fn render_mime_message_and_pre_message( + context: &Context, + msg: &mut Message, + mimefactory: MimeFactory, +) -> Result<(RenderedEmail, Option)> { + let needs_pre_message = msg.viewtype.has_file() + && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages + && msg + .get_filebytes(context) + .await? + .context("filebytes not available, even though message has attachment")? + > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; + + if needs_pre_message { + info!( + context, + "Message is large and will be split into a pre- and a post-message.", + ); + + let mut mimefactory_post_msg = mimefactory.clone(); + mimefactory_post_msg.set_as_post_message(); + let rendered_msg = mimefactory_post_msg.render(context).await?; + + let mut mimefactory_pre_msg = mimefactory; + mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + let rendered_pre_msg = mimefactory_pre_msg + .render(context) + .await + .context("pre-message failed to render")?; + + if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { + warn!( + context, + "Pre-message for message (MsgId={}) is larger than expected: {}.", + msg.id, + rendered_pre_msg.message.len() + ); + } + + Ok((rendered_msg, Some(rendered_pre_msg))) + } else { + Ok((mimefactory.render(context).await?, None)) + } +} + /// Constructs jobs for sending a message and inserts them into the `smtp` table. /// /// Updates the message `GuaranteeE2ee` parameter and persists it @@ -2807,13 +2861,29 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let rendered_msg = match mimefactory.render(context).await { - Ok(res) => Ok(res), - Err(err) => { - message::set_msg_failed(context, msg, &err.to_string()).await?; - Err(err) - } - }?; + let (rendered_msg, rendered_pre_msg) = + match render_mime_message_and_pre_message(context, msg, mimefactory).await { + Ok(res) => Ok(res), + Err(err) => { + message::set_msg_failed(context, msg, &err.to_string()).await?; + Err(err) + } + }?; + + if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) { + info!( + context, + "Message Sizes: Pre-Message {}; Post-Message: {}", + format_size(pre_msg.message.len(), BINARY), + format_size(post_msg.message.len(), BINARY) + ); + } else { + info!( + context, + "Message will be sent as normal message (no pre- and post message). Size: {}", + format_size(rendered_msg.message.len(), BINARY) + ); + } if needs_encryption && !rendered_msg.is_encrypted { /* unrecoverable */ @@ -2867,19 +2937,28 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - (), )?; } - + let mut stmt = t.prepare( + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) + VALUES (?1, ?2, ?3, ?4)", + )?; for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); - let row_id = t.execute( - "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ - VALUES (?1, ?2, ?3, ?4)", - ( - &rendered_msg.rfc724_mid, - recipients_chunk, - &rendered_msg.message, + // send pre-message before actual message + if let Some(pre_msg) = &rendered_pre_msg { + let row_id = stmt.execute(( + &pre_msg.rfc724_mid, + &recipients_chunk, + &pre_msg.message, msg.id, - ), - )?; + ))?; + row_ids.push(row_id.try_into()?); + } + let row_id = stmt.execute(( + &rendered_msg.rfc724_mid, + &recipients_chunk, + &rendered_msg.message, + msg.id, + ))?; row_ids.push(row_id.try_into()?); } Ok(row_ids) @@ -4261,6 +4340,14 @@ pub async fn forward_msgs_2ctx( msg.viewtype = Viewtype::Text; } + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } + let param = &mut param; msg.param.steal(param, Param::File); msg.param.steal(param, Param::Filename); @@ -4337,12 +4424,22 @@ pub(crate) async fn save_copy_in_self_talk( msg.param.remove(Param::WebxdcDocumentTimestamp); msg.param.remove(Param::WebxdcSummary); msg.param.remove(Param::WebxdcSummaryTimestamp); + msg.param.remove(Param::PostMessageFileBytes); + msg.param.remove(Param::PostMessageViewtype); + + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } if !msg.original_msg_id.is_unset() { bail!("message already saved."); } - let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt, + let copy_fields = "from_id, to_id, timestamp_rcvd, type, mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg"; let row_id = context .sql @@ -4350,7 +4447,7 @@ pub(crate) async fn save_copy_in_self_talk( &format!( "INSERT INTO msgs ({copy_fields}, timestamp_sent, - chat_id, rfc724_mid, state, timestamp, param, starred) + txt, chat_id, rfc724_mid, state, timestamp, param, starred) SELECT {copy_fields}, -- Outgoing messages on originating device -- have timestamp_sent == 0. @@ -4358,10 +4455,11 @@ pub(crate) async fn save_copy_in_self_talk( -- so UIs display the same timestamp -- for saved and original message. IIF(timestamp_sent == 0, timestamp, timestamp_sent), - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ? FROM msgs WHERE id=?;" ), ( + msg.text, dest_chat_id, dest_rfc724_mid, if msg.from_id == ContactId::SELF { diff --git a/src/chat/chat_tests.rs b/src/chat/chat_tests.rs index a93ab2179a..1e8cff82fb 100644 --- a/src/chat/chat_tests.rs +++ b/src/chat/chat_tests.rs @@ -3116,7 +3116,7 @@ async fn test_broadcast_channel_protected_listid() -> Result<()> { .await? .grpid; - let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes(), None).await?; + let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes()).await?; assert_eq!( parsed.get_mailinglist_header().unwrap(), format!("My Channel <{}>", alice_list_id) @@ -3311,7 +3311,7 @@ async fn test_leave_broadcast_multidevice() -> Result<()> { remove_contact_from_chat(bob0, bob_chat_id, ContactId::SELF).await?; let leave_msg = bob0.pop_sent_msg().await; - let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes(), None).await?; + let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes()).await?; assert_eq!(parsed.parts[0].msg, "I left the group."); let rcvd = bob1.recv_msg(&leave_msg).await; diff --git a/src/config.rs b/src/config.rs index e719ac3892..350e99ea35 100644 --- a/src/config.rs +++ b/src/config.rs @@ -438,8 +438,8 @@ pub enum Config { /// using this still run unmodified code. TestHooks, - /// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests. - FailOnReceivingFullMsg, + /// Return an error from `receive_imf_inner()`. For tests. + SimulateReceiveImfError, /// Enable composing emails with Header Protection as defined in /// "Header Protection for Cryptographically diff --git a/src/context.rs b/src/context.rs index 12c3f645d1..90e480bd79 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1092,13 +1092,6 @@ impl Context { .await? .unwrap_or_default(), ); - res.insert( - "fail_on_receiving_full_msg", - self.sql - .get_raw_config("fail_on_receiving_full_msg") - .await? - .unwrap_or_default(), - ); res.insert( "std_header_protection_composing", self.sql diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 4a20c3af37..c4ffc1648a 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -297,6 +297,7 @@ async fn test_get_info_completeness() { "encrypted_device_token", "stats_last_update", "stats_last_old_contact_id", + "simulate_receive_imf_error", // only used in tests ]; let t = TestContext::new().await; let info = t.get_info().await.unwrap(); diff --git a/src/download.rs b/src/download.rs index cddd0fff4c..f49fe301d5 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,27 +1,18 @@ //! # Download large messages manually. -use std::cmp::max; use std::collections::BTreeMap; use anyhow::{Result, anyhow, bail, ensure}; use deltachat_derive::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use crate::config::Config; use crate::context::Context; use crate::imap::session::Session; -use crate::message::{Message, MsgId, Viewtype}; -use crate::mimeparser::{MimeMessage, Part}; -use crate::tools::time; -use crate::{EventType, chatlist_events, stock_str}; +use crate::log::warn; +use crate::message::{self, Message, MsgId, rfc724_mid_exists}; +use crate::{EventType, chatlist_events}; -/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`. -/// -/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN) -/// should always be downloaded completely to handle them correctly, -/// also in larger groups and if group and contact avatar are attached. -/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`. -pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; +pub(crate) mod pre_msg_metadata; /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), @@ -29,6 +20,15 @@ pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case. pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; +/// From this point onward outgoing messages are considered large +/// and get a Pre-Message, which announces the Post-Message. +// this is only about sending so we can modify it any time. +// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB) +pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; + +/// Max size for pre messages. A warning is emitted when this is exceeded. +pub(crate) const PRE_MSG_SIZE_WARNING_THRESHOLD: usize = 150_000; + /// Download state of the message. #[derive( Debug, @@ -64,20 +64,8 @@ pub enum DownloadState { InProgress = 1000, } -impl Context { - // Returns validated download limit or `None` for "no limit". - pub(crate) async fn download_limit(&self) -> Result> { - let download_limit = self.get_config_int(Config::DownloadLimit).await?; - if download_limit <= 0 { - Ok(None) - } else { - Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32))) - } - } -} - impl MsgId { - /// Schedules full message download for partially downloaded message. + /// Schedules Post-Message download for partially downloaded message. pub async fn download_full(self, context: &Context) -> Result<()> { let msg = Message::load_from_db(context, self).await?; match msg.download_state() { @@ -86,11 +74,17 @@ impl MsgId { } DownloadState::InProgress => return Err(anyhow!("Download already in progress.")), DownloadState::Available | DownloadState::Failure => { + if msg.rfc724_mid().is_empty() { + return Err(anyhow!("Download not possible, message has no rfc724_mid")); + } self.update_download_state(context, DownloadState::InProgress) .await?; context .sql - .execute("INSERT INTO download (msg_id) VALUES (?)", (self,)) + .execute( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,?)", + (msg.rfc724_mid(), msg.id), + ) .await?; context.scheduler.interrupt_inbox().await; } @@ -139,20 +133,9 @@ impl Message { /// Most messages are downloaded automatically on fetch instead. pub(crate) async fn download_msg( context: &Context, - msg_id: MsgId, + rfc724_mid: String, session: &mut Session, ) -> Result<()> { - let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else { - // If partially downloaded message was already deleted - // we do not know its Message-ID anymore - // so cannot download it. - // - // Probably the message expired due to `delete_device_after` - // setting or was otherwise removed from the device, - // so we don't want it to reappear anyway. - return Ok(()); - }; - let transport_id = session.transport_id(); let row = context .sql @@ -161,7 +144,7 @@ pub(crate) async fn download_msg( WHERE rfc724_mid=? AND transport_id=? AND target!=''", - (&msg.rfc724_mid, transport_id), + (&rfc724_mid, transport_id), |row| { let server_uid: u32 = row.get(0)?; let server_folder: String = row.get(1)?; @@ -176,7 +159,7 @@ pub(crate) async fn download_msg( }; session - .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone()) + .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid) .await?; Ok(()) } @@ -209,7 +192,7 @@ impl Session { let mut uid_message_ids: BTreeMap = BTreeMap::new(); uid_message_ids.insert(uid, rfc724_mid); let (sender, receiver) = async_channel::unbounded(); - self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, sender) + self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender) .await?; if receiver.recv().await.is_err() { bail!("Failed to fetch UID {uid}"); @@ -218,41 +201,143 @@ impl Session { } } -impl MimeMessage { - /// Creates a placeholder part and add that to `parts`. - /// - /// To create the placeholder, only the outermost header can be used, - /// the mime-structure itself is not available. - /// - /// The placeholder part currently contains a text with size and availability of the message. - pub(crate) async fn create_stub_from_partial_download( - &mut self, - context: &Context, - org_bytes: u32, - ) -> Result<()> { - let mut text = format!( - "[{}]", - stock_str::partial_download_msg_body(context, org_bytes).await - ); - if let Some(delete_server_after) = context.get_config_delete_server_after().await? { - let until = stock_str::download_availability( - context, - time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER), - ) - .await; - text += format!(" [{until}]").as_str(); - }; +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} - info!(context, "Partial download: {}", text); +async fn available_post_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} - self.do_add_single_part(Part { - typ: Viewtype::Text, - msg: text, - ..Default::default() - }); +async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} - Ok(()) +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +// this is a dedicated method because it is used in multiple places. +pub(crate) async fn premessage_is_downloaded_for( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(message::rfc724_mid_exists(context, rfc724_mid) + .await? + .is_some()) +} + +pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + warn!( + context, + "{rfc724_mid} is in available_post_msgs table but we failed to fetch it, + so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime" + ); + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } + } } + + Ok(()) +} + +/// Download known post messages without pre_message +/// in order to guard against lost pre-messages: +pub(crate) async fn download_known_post_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the Post-Message unconditionally, + // because the Pre-Message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_post_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) } #[cfg(test)] @@ -260,11 +345,8 @@ mod tests { use num_traits::FromPrimitive; use super::*; - use crate::chat::{get_chat_msgs, send_msg}; - use crate::ephemeral::Timer; - use crate::message::delete_msgs; - use crate::receive_imf::receive_imf_from_inbox; - use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; + use crate::chat::send_msg; + use crate::test_utils::TestContext; #[test] fn test_downloadstate_values() { @@ -282,29 +364,6 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_download_limit() -> Result<()> { - let t = TestContext::new_alice().await; - - assert_eq!(t.download_limit().await?, None); - - t.set_config(Config::DownloadLimit, Some("200000")).await?; - assert_eq!(t.download_limit().await?, Some(200000)); - - t.set_config(Config::DownloadLimit, Some("20000")).await?; - assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - t.set_config(Config::DownloadLimit, None).await?; - assert_eq!(t.download_limit().await?, None); - - for val in &["0", "-1", "-100", "", "foo"] { - t.set_config(Config::DownloadLimit, Some(val)).await?; - assert_eq!(t.download_limit().await?, None); - } - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_download_state() -> Result<()> { let t = TestContext::new_alice().await; @@ -336,230 +395,4 @@ mod tests { Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_receive_imf() -> Result<()> { - let t = TestContext::new_alice().await; - - let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ - From: bob@example.com\n\ - To: alice@example.org\n\ - Subject: foo\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\ - Content-Type: text/plain"; - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - header.as_bytes(), - false, - Some(100000), - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Available); - assert_eq!(msg.get_subject(), "foo"); - assert!( - msg.get_text() - .contains(&stock_str::partial_download_msg_body(&t, 100000).await) - ); - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - format!("{header}\n\n100k text...").as_bytes(), - false, - None, - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Done); - assert_eq!(msg.get_subject(), "foo"); - assert_eq!(msg.get_text(), "100k text..."); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_ephemeral() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = t - .create_chat_with_contact("bob", "bob@example.org") - .await - .id; - chat_id - .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 }) - .await?; - - // download message from bob partially, this must not change the ephemeral timer - receive_imf_from_inbox( - &t, - "first@example.org", - b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain", - false, - Some(100000), - ) - .await?; - assert_eq!( - chat_id.get_ephemeral_timer(&t).await?, - Timer::Enabled { duration: 60 } - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_status_update_expands_to_nothing() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat_id = alice.create_chat(&bob).await.id; - - let file = alice.get_blobdir().join("minimal.xdc"); - tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?; - let mut instance = Message::new(Viewtype::File); - instance.set_file_and_deduplicate(&alice, &file, None, None)?; - let _sent1 = alice.send_msg(chat_id, &mut instance).await; - - alice - .send_webxdc_status_update(instance.id, r#"{"payload":7}"#) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid; - - // not downloading the status update results in an placeholder - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - Some(sent2.payload().len() as u32), - ) - .await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!( - get_chat_msgs(&bob, chat_id).await?.len(), - E2EE_INFO_MSGS + 1 - ); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat - // (usually status updates are too small for not being downloaded directly) - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - None, - ) - .await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), E2EE_INFO_MSGS); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_mdn_expands_to_nothing() -> Result<()> { - let bob = TestContext::new_bob().await; - let raw = b"Subject: Message opened\n\ - Date: Mon, 10 Jan 2020 00:00:00 +0000\n\ - Chat-Version: 1.0\n\ - Message-ID: \n\ - To: Alice \n\ - From: Bob \n\ - Content-Type: multipart/report; report-type=disposition-notification;\n\t\ - boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: text/plain; charset=utf-8\n\ - \n\ - bla\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: message/disposition-notification\n\ - \n\ - Reporting-UA: Delta Chat 1.88.0\n\ - Original-Recipient: rfc822;bob@example.org\n\ - Final-Recipient: rfc822;bob@example.org\n\ - Original-Message-ID: \n\ - Disposition: manual-action/MDN-sent-automatically; displayed\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ - "; - - // not downloading the mdn results in an placeholder - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, Some(raw.len() as u32)).await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the mdn afterwards expands to nothing and deletes the placeholder directly - // (usually mdn are too small for not being downloaded directly) - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None).await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - /// Tests that fully downloading the message - /// works even if the Message-ID already exists - /// in the database assigned to the trash chat. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_trashed() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let imf_raw = b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - - // Download message from Bob partially. - let partial_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, Some(100000)) - .await? - .unwrap(); - assert_eq!(partial_received_msg.msg_ids.len(), 1); - - // Delete the received message. - // Not it is still in the database, - // but in the trash chat. - delete_msgs(alice, &[partial_received_msg.msg_ids[0]]).await?; - - // Fully download message after deletion. - let full_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, None).await?; - - // The message does not reappear. - // However, `receive_imf` should not fail. - assert!(full_received_msg.is_none()); - - Ok(()) - } } diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs new file mode 100644 index 0000000000..cc19b9e1f7 --- /dev/null +++ b/src/download/pre_msg_metadata.rs @@ -0,0 +1,248 @@ +use anyhow::{Context as _, Result}; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::context::Context; +use crate::log::warn; +use crate::message::Message; +use crate::message::Viewtype; +use crate::param::{Param, Params}; + +/// Metadata contained in Pre-Message that describes the Post-Message. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PreMsgMetadata { + /// size of the attachment in bytes + pub(crate) size: u64, + /// Real viewtype of message + pub(crate) viewtype: Viewtype, + /// the original file name + pub(crate) filename: String, + /// Dimensions: width and height of image or video + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) dimensions: Option<(i32, i32)>, + /// Duration of audio file or video in milliseconds + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) duration: Option, +} + +impl PreMsgMetadata { + // Returns PreMsgMetadata for messages with files and None for messages without file attachment + pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result> { + if !message.viewtype.has_file() { + return Ok(None); + } + + let size = message + .get_filebytes(context) + .await? + .context("Unexpected: file has no size")?; + let filename = message + .param + .get(Param::Filename) + .unwrap_or_default() + .to_owned(); + let dimensions = { + match ( + message.param.get_int(Param::Width), + message.param.get_int(Param::Height), + ) { + (None, None) => None, + (Some(width), Some(height)) => Some((width, height)), + _ => { + warn!(context, "Message misses either width or height."); + None + } + } + }; + let duration = message.param.get_int(Param::Duration); + + Ok(Some(Self { + size, + filename, + viewtype: message.viewtype, + dimensions, + duration, + })) + } + + pub(crate) fn to_header_value(&self) -> Result { + Ok(serde_json::to_string(&self)?) + } + + pub(crate) fn try_from_header_value(value: &str) -> Result { + Ok(serde_json::from_str(value)?) + } +} + +impl Params { + /// Applies data from pre_msg_metadata to Params + pub(crate) fn apply_from_pre_msg_metadata( + &mut self, + pre_msg_metadata: &PreMsgMetadata, + ) -> &mut Self { + self.set(Param::PostMessageFileBytes, pre_msg_metadata.size); + if !pre_msg_metadata.filename.is_empty() { + self.set(Param::Filename, &pre_msg_metadata.filename); + } + self.set_i64( + Param::PostMessageViewtype, + pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), + ); + if let Some((width, height)) = pre_msg_metadata.dimensions { + self.set(Param::Width, width); + self.set(Param::Height, height); + } + if let Some(duration) = pre_msg_metadata.duration { + self.set(Param::Duration, duration); + } + + self + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use pretty_assertions::assert_eq; + + use crate::{ + message::{Message, Viewtype}, + test_utils::{TestContextManager, create_test_image}, + }; + + use super::PreMsgMetadata; + + /// Build from message with file attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_file_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let mut file_msg = Message::new(Viewtype::File); + file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + }) + ); + Ok(()) + } + + /// Build from message with image attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_image_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let mut image_msg = Message::new(Viewtype::Image); + + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?; + // this is usually done while sending, + // but we don't send it here, so we need to call it ourself + image_msg.try_calc_and_set_dimensions(alice).await?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1816098, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((width as i32, height as i32)), + duration: None, + }) + ); + + Ok(()) + } + + /// Test that serialisation results in expected format + #[test] + fn test_serialize_to_header() -> Result<()> { + assert_eq!( + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + .to_header_value()?, + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + .to_header_value()?, + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + .to_header_value()?, + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + ); + + Ok(()) + } + + /// Test that deserialisation from expected format works + /// This test will become important for compatibility between versions in the future + #[test] + fn test_deserialize_from_header() -> Result<()> { + assert_eq!( + serde_json::from_str::( + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" + )?, + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + )?, + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + )?, + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + ); + + Ok(()) + } +} diff --git a/src/headerdef.rs b/src/headerdef.rs index c57f05033f..4baee1c57c 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,6 +102,21 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// A message with a large attachment is split into two MIME messages: + /// A pre-message, which contains everything but the attachment, + /// and a Post-Message. + /// The Pre-Message gets a `Chat-Post-Message-Id` header + /// referencing the Post-Message's rfc724_mid. + ChatPostMessageId, + + /// Announce Post-Message metadata in a Pre-Message. + /// contains serialized PreMsgMetadata struct + ChatPostMessageMetadata, + + /// This message is preceded by a Pre-Message + /// and thus this message can be skipped while fetching messages. + /// This is a cleartext / unproteced header. + ChatIsPostMessage, /// [Autocrypt](https://autocrypt.org/) header. Autocrypt, @@ -147,6 +162,9 @@ pub enum HeaderDef { impl HeaderDef { /// Returns the corresponding header string. + /// + /// Format is lower-kebab-case for easy comparisons. + /// This method is used in message receiving and testing. pub fn get_headername(&self) -> &'static str { self.into() } diff --git a/src/imap.rs b/src/imap.rs index 0806c14a98..b2150d3ba6 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -67,7 +67,6 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\ X-MICROSOFT-ORIGINAL-MESSAGE-ID\ )])"; const BODY_FULL: &str = "(FLAGS BODY.PEEK[])"; -const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])"; #[derive(Debug)] pub(crate) struct Imap { @@ -615,12 +614,23 @@ impl Imap { .context("prefetch")?; let read_cnt = msgs.len(); - let download_limit = context.download_limit().await?; - let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1); + let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); + let mut available_post_msgs = Vec::::with_capacity(msgs.len()); + let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; + let download_limit = { + let download_limit: Option = + context.get_config_parsed(Config::DownloadLimit).await?; + if download_limit == Some(0) { + None + } else { + download_limit + } + }; + // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { let headers = match get_fetch_headers(fetch_response) { @@ -632,6 +642,9 @@ impl Imap { }; let message_id = prefetch_get_message_id(&headers); + let size = fetch_response + .size + .context("imap fetch response does not contain size")?; // Determine the target folder where the message should be moved to. // @@ -706,14 +719,23 @@ impl Imap { ) .await.context("prefetch_should_download")? { - match download_limit { - Some(download_limit) => uids_fetch.push(( - uid, - fetch_response.size.unwrap_or_default() > download_limit, - )), - None => uids_fetch.push((uid, false)), - } - uid_message_ids.insert(uid, message_id); + if headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + info!(context, "{message_id:?} is a post-message."); + available_post_msgs.push(message_id.clone()); + + // whether it fits download size limit + if download_limit.is_none_or(|download_limit| size < download_limit) { + download_when_normal_starts.push(message_id.clone()); + } + } else { + info!(context, "{message_id:?} is not a post-message."); + + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + }; } else { largest_uid_skipped = Some(uid); } @@ -747,29 +769,10 @@ impl Imap { }; let actually_download_messages_future = async { - let sender = sender; - let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1)); - let mut fetch_partially = false; - uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1)); - for (uid, fp) in uids_fetch { - if fp != fetch_partially { - session - .fetch_many_msgs( - context, - folder, - uids_fetch_in_batch.split_off(0), - &uid_message_ids, - fetch_partially, - sender.clone(), - ) - .await - .context("fetch_many_msgs")?; - fetch_partially = fp; - } - uids_fetch_in_batch.push(uid); - } - - anyhow::Ok(()) + session + .fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender) + .await + .context("fetch_many_msgs") }; let (largest_uid_fetched, fetch_res) = @@ -804,6 +807,30 @@ impl Imap { chat::mark_old_messages_as_noticed(context, received_msgs).await?; + if fetch_res.is_ok() { + info!( + context, + "available_post_msgs: {}, download_when_normal_starts: {}", + available_post_msgs.len(), + download_when_normal_starts.len() + ); + for rfc724_mid in available_post_msgs { + context + .sql + .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) + .await?; + } + for rfc724_mid in download_when_normal_starts { + context + .sql + .insert( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)", + (rfc724_mid,), + ) + .await?; + } + } + // Now fail if fetching failed, so we will // establish a new session if this one is broken. fetch_res?; @@ -1329,7 +1356,6 @@ impl Session { folder: &str, request_uids: Vec, uid_message_ids: &BTreeMap, - fetch_partially: bool, received_msgs_channel: Sender<(u32, Option)>, ) -> Result<()> { if request_uids.is_empty() { @@ -1337,25 +1363,10 @@ impl Session { } for (request_uids, set) in build_sequence_sets(&request_uids)? { - info!( - context, - "Starting a {} FETCH of message set \"{}\".", - if fetch_partially { "partial" } else { "full" }, - set - ); - let mut fetch_responses = self - .uid_fetch( - &set, - if fetch_partially { - BODY_PARTIAL - } else { - BODY_FULL - }, - ) - .await - .with_context(|| { - format!("fetching messages {} from folder \"{}\"", &set, folder) - })?; + info!(context, "Starting a full FETCH of message set \"{}\".", set); + let mut fetch_responses = self.uid_fetch(&set, BODY_FULL).await.with_context(|| { + format!("fetching messages {} from folder \"{}\"", &set, folder) + })?; // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here // when we want to process other messages first. @@ -1412,11 +1423,7 @@ impl Session { count += 1; let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted); - let (body, partial) = if fetch_partially { - (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ... - } else { - (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header() - }; + let body = fetch_response.body(); if is_deleted { info!(context, "Not processing deleted msg {}.", request_uid); @@ -1450,7 +1457,7 @@ impl Session { context, "Passing message UID {} to receive_imf().", request_uid ); - let res = receive_imf_inner(context, rfc724_mid, body, is_seen, partial).await; + let res = receive_imf_inner(context, rfc724_mid, body, is_seen).await; let received_msg = match res { Err(err) => { warn!(context, "receive_imf error: {err:#}."); diff --git a/src/imap/session.rs b/src/imap/session.rs index 0da1d7936f..1d8c2d7110 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,6 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. +/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ @@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE FROM \ IN-REPLY-TO REFERENCES \ CHAT-VERSION \ + CHAT-IS-POST-MESSAGE \ AUTO-SUBMITTED \ AUTOCRYPT-SETUP-MESSAGE\ )])"; diff --git a/src/internals_for_benches.rs b/src/internals_for_benches.rs index 1fd80c5516..61fc9d7fca 100644 --- a/src/internals_for_benches.rs +++ b/src/internals_for_benches.rs @@ -21,7 +21,7 @@ pub async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result< } pub async fn parse_and_get_text(context: &Context, imf_raw: &[u8]) -> Result { - let mime_parser = MimeMessage::from_bytes(context, imf_raw, None).await?; + let mime_parser = MimeMessage::from_bytes(context, imf_raw).await?; Ok(mime_parser.parts.into_iter().next().unwrap().msg) } diff --git a/src/message.rs b/src/message.rs index f02aa2619b..ea29a9b67d 100644 --- a/src/message.rs +++ b/src/message.rs @@ -8,6 +8,9 @@ use std::str; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; use deltachat_derive::{FromSql, ToSql}; +use humansize::BINARY; +use humansize::format_size; +use num_traits::FromPrimitive; use serde::{Deserialize, Serialize}; use tokio::{fs, io}; @@ -430,6 +433,10 @@ pub struct Message { pub(crate) ephemeral_timer: EphemeralTimer, pub(crate) ephemeral_timestamp: i64, pub(crate) text: String, + /// Text that is added to the end of Message.text + /// + /// Currently used for adding the download information on pre-messages + pub(crate) additional_text: String, /// Message subject. /// @@ -488,7 +495,7 @@ impl Message { !id.is_special(), "Can not load special message ID {id} from DB" ); - let msg = context + let mut msg = context .sql .query_row_optional( concat!( @@ -570,6 +577,7 @@ impl Message { original_msg_id: row.get("original_msg_id")?, mime_modified: row.get("mime_modified")?, text, + additional_text: String::new(), subject: row.get("subject")?, param: row.get::<_, String>("param")?.parse().unwrap_or_default(), hidden: row.get("hidden")?, @@ -584,9 +592,48 @@ impl Message { .await .with_context(|| format!("failed to load message {id} from the database"))?; + if let Some(msg) = &mut msg { + msg.additional_text = + Self::get_additional_text(context, msg.download_state, &msg.param).await?; + } + Ok(msg) } + /// Returns additional text which is appended to the message's text field + /// when it is loaded from the database. + /// Currently this is used to add infomation to pre-messages of what the download will be and how large it is + async fn get_additional_text( + context: &Context, + download_state: DownloadState, + param: &Params, + ) -> Result { + if download_state != DownloadState::Done { + let file_size = param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + .map(|file_size: usize| format_size(file_size, BINARY)) + .unwrap_or("?".to_owned()); + let viewtype = param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + .unwrap_or(Viewtype::Unknown); + let file_name = param + .get(Param::Filename) + .map(sanitize_filename) + .unwrap_or("?".to_owned()); + + return match viewtype { + Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")), + _ => { + let translated_viewtype = viewtype.to_locale_string(context).await; + Ok(format!(" [{translated_viewtype} - {file_size}]")) + } + }; + } + Ok(String::new()) + } + /// Returns the MIME type of an attached file if it exists. /// /// If the MIME type is not known, the function guesses the MIME type @@ -769,7 +816,7 @@ impl Message { /// Returns the text of the message. pub fn get_text(&self) -> String { - self.text.clone() + self.text.clone() + &self.additional_text } /// Returns message subject. @@ -791,7 +838,17 @@ impl Message { } /// Returns the size of the file in bytes, if applicable. + /// If message is a pre-message, then this returns size of the to be downloaded file. pub async fn get_filebytes(&self, context: &Context) -> Result> { + // if download state is not downloaded then return value from from params metadata + if self.download_state != DownloadState::Done + && let Some(file_size) = self + .param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + { + return Ok(Some(file_size)); + } if let Some(path) = self.param.get_file_path(context)? { Ok(Some(get_filebytes(context, &path).await.with_context( || format!("failed to get {} size in bytes", path.display()), @@ -801,6 +858,21 @@ impl Message { } } + /// If message is a Pre-Message, + /// then this returns the viewtype it will have when it is downloaded. + #[cfg(test)] + pub(crate) fn get_post_message_viewtype(&self) -> Option { + if self.download_state != DownloadState::Done + && let Some(viewtype) = self + .param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + { + return Some(viewtype); + } + None + } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() @@ -1681,9 +1753,17 @@ pub async fn delete_msgs_ex( let update_db = |trans: &mut rusqlite::Transaction| { trans.execute( "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, msg.rfc724_mid), + (target, &msg.rfc724_mid), )?; trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; + trans.execute( + "DELETE FROM download WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; + trans.execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; Ok(()) }; if let Err(e) = context.sql.transaction(update_db).await { @@ -1752,7 +1832,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> "SELECT m.chat_id AS chat_id, m.state AS state, - m.download_state as download_state, m.ephemeral_timer AS ephemeral_timer, m.param AS param, m.from_id AS from_id, @@ -1765,7 +1844,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> |row| { let chat_id: ChatId = row.get("chat_id")?; let state: MessageState = row.get("state")?; - let download_state: DownloadState = row.get("download_state")?; let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default(); let from_id: ContactId = row.get("from_id")?; let rfc724_mid: String = row.get("rfc724_mid")?; @@ -1777,7 +1855,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, chat_id, state, - download_state, param, from_id, rfc724_mid, @@ -1810,7 +1887,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, curr_chat_id, curr_state, - curr_download_state, curr_param, curr_from_id, curr_rfc724_mid, @@ -1820,14 +1896,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> _curr_ephemeral_timer, ) in msgs { - if curr_download_state != DownloadState::Done { - if curr_state == MessageState::InFresh { - // Don't mark partially downloaded messages as seen or send a read receipt since - // they are not really seen by the user. - update_msg_state(context, id, MessageState::InNoticed).await?; - updated_chat_ids.insert(curr_chat_id); - } - } else if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { + if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { update_msg_state(context, id, MessageState::InSeen).await?; info!(context, "Seen message {}.", id); diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index be3ed01d9a..d43d21220e 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -326,112 +326,6 @@ async fn test_markseen_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_markseen_not_downloaded_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = bob.create_chat(alice).await.id; - alice.create_chat(bob).await; // Make sure the chat is accepted. - - tcm.section("Bob sends a large message to Alice"); - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - - tcm.section("Alice receives a large message from Bob"); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert!(!msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert_eq!(msg.state, MessageState::InFresh); - markseen_msgs(alice, vec![msg.id]).await?; - // A not downloaded message can be seen only if it's seen on another device. - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - // Marking the message as seen again is a no op. - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - msg.id - .update_download_state(alice, DownloadState::InProgress) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Failure) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Undecipherable) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - assert!( - !alice - .sql - .exists("SELECT COUNT(*) FROM smtp_mdns", ()) - .await? - ); - - alice.set_config(Config::DownloadLimit, None).await?; - // Let's assume that Alice and Bob resolved the problem with encryption. - let old_msg = msg; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.chat_id, old_msg.chat_id); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - // The message state mustn't be downgraded to `InFresh`. - assert_eq!(msg.state, MessageState::InNoticed); - markseen_msgs(alice, vec![msg.id]).await?; - let msg = Message::load_from_db(alice, msg.id).await?; - assert_eq!(msg.state, MessageState::InSeen); - assert_eq!( - alice - .sql - .count("SELECT COUNT(*) FROM smtp_mdns", ()) - .await?, - 1 - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = tcm.send_recv_accept(alice, bob, "hi").await.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - alice.set_config(Config::DownloadLimit, None).await?; - let seen = true; - let rcvd_msg = receive_imf(alice, sent_msg.payload().as_bytes(), seen) - .await - .unwrap() - .unwrap(); - assert_eq!(rcvd_msg.chat_id, msg.chat_id); - let msg = Message::load_from_db(alice, *rcvd_msg.msg_ids.last().unwrap()) - .await - .unwrap(); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - assert_eq!(msg.state, MessageState::InSeen); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_state() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 732fff545a..049114847a 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG}; use crate::constants::{Chattype, DC_FROM_HANDSHAKE}; use crate::contact::{Contact, ContactId, Origin}; use crate::context::Context; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::e2ee::EncryptHelper; use crate::ensure_and_debug_assert; use crate::ephemeral::Timer as EphemeralTimer; @@ -59,6 +60,15 @@ pub enum Loaded { }, } +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// adds the Chat-Is-Post-Message header in unprotected part + PostMessage, + /// adds the Chat-Post-Message-ID header to protected part + /// also adds metadata and explicitly excludes attachment + PreMessage { post_msg_rfc724_mid: String }, +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -146,6 +156,9 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, + + /// This field is used when this is either a pre-message or a Post-Message. + pre_message_mode: Option, } /// Result of rendering a message, ready to be submitted to a send job. @@ -500,6 +513,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, + pre_message_mode: None, }; Ok(factory) } @@ -548,6 +562,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, + pre_message_mode: None, }; Ok(res) @@ -779,7 +794,10 @@ impl MimeFactory { headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into())); let rfc724_mid = match &self.loaded { - Loaded::Message { msg, .. } => msg.rfc724_mid.clone(), + Loaded::Message { msg, .. } => match &self.pre_message_mode { + Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + _ => msg.rfc724_mid.clone(), + }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), }; headers.push(( @@ -893,7 +911,7 @@ impl MimeFactory { )); } - let is_encrypted = self.encryption_pubkeys.is_some(); + let is_encrypted = self.will_be_encrypted(); // Add ephemeral timer for non-MDN messages. // For MDNs it does not matter because they are not visible @@ -978,6 +996,22 @@ impl MimeFactory { "MIME-Version", mail_builder::headers::raw::Raw::new("1.0").into(), )); + + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + unprotected_headers.push(( + "Chat-Is-Post-Message", + mail_builder::headers::raw::Raw::new("1").into(), + )); + } else if let Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + }) = self.pre_message_mode.clone() + { + protected_headers.push(( + "Chat-Post-Message-ID", + mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(), + )); + } + for header @ (original_header_name, _header_value) in &headers { let header_name = original_header_name.to_lowercase(); if header_name == "message-id" { @@ -1119,6 +1153,10 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + continue; + } + let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup || cmd == SystemMessage::SecurejoinMessage || multiple_recipients && { @@ -1831,19 +1869,23 @@ impl MimeFactory { let footer = if is_reaction { "" } else { &self.selfstatus }; - let message_text = format!( - "{}{}{}{}{}{}", - fwdhint.unwrap_or_default(), - quoted_text.unwrap_or_default(), - escape_message_footer_marks(final_text), - if !final_text.is_empty() && !footer.is_empty() { - "\r\n\r\n" - } else { - "" - }, - if !footer.is_empty() { "-- \r\n" } else { "" }, - footer - ); + let message_text = if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + "".to_string() + } else { + format!( + "{}{}{}{}{}{}", + fwdhint.unwrap_or_default(), + quoted_text.unwrap_or_default(), + escape_message_footer_marks(final_text), + if !final_text.is_empty() && !footer.is_empty() { + "\r\n\r\n" + } else { + "" + }, + if !footer.is_empty() { "-- \r\n" } else { "" }, + footer + ) + }; let mut main_part = MimePart::new("text/plain", message_text); if is_reaction { @@ -1875,8 +1917,19 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - let file_part = build_body_file(context, &msg).await?; - parts.push(file_part); + if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { + bail!("Failed to generate metadata for pre-message") + }; + + headers.push(( + HeaderDef::ChatPostMessageMetadata.into(), + mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(), + )); + } else { + let file_part = build_body_file(context, &msg).await?; + parts.push(file_part); + } } if let Some(msg_kml_part) = self.get_message_kml_part() { @@ -1921,6 +1974,8 @@ impl MimeFactory { } } + self.attach_selfavatar = + self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage); if self.attach_selfavatar { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { @@ -1990,6 +2045,20 @@ impl MimeFactory { Ok(message) } + + pub fn will_be_encrypted(&self) -> bool { + self.encryption_pubkeys.is_some() + } + + pub fn set_as_post_message(&mut self) { + self.pre_message_mode = Some(PreMessageMode::PostMessage); + } + + pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) { + self.pre_message_mode = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid: post_message.rfc724_mid.clone(), + }); + } } fn hidden_recipients() -> Address<'static> { diff --git a/src/mimefactory/mimefactory_tests.rs b/src/mimefactory/mimefactory_tests.rs index d29fe54645..bd79d92c5f 100644 --- a/src/mimefactory/mimefactory_tests.rs +++ b/src/mimefactory/mimefactory_tests.rs @@ -559,7 +559,7 @@ async fn test_render_reply() { "1.0" ); - let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes(), None) + let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes()) .await .unwrap(); } @@ -757,7 +757,7 @@ async fn test_protected_headers_directive() -> Result<()> { assert!(msg.get_showpadlock()); assert!(sent.payload.contains("\r\nSubject: [...]\r\n")); - let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?; + let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes()).await?; let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n"); let part = payload.next().unwrap(); assert_eq!( @@ -781,7 +781,7 @@ async fn test_hp_outer_headers() -> Result<()> { .await?; chat::send_text_msg(t, chat_id, "hi!".to_string()).await?; let sent_msg = t.pop_sent_msg().await; - let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes()).await?; assert_eq!(msg.header_exists(HeaderDef::HpOuter), std_hp_composing); for hdr in ["Date", "From", "Message-ID"] { assert_eq!( @@ -811,7 +811,7 @@ async fn test_dont_remove_self() -> Result<()> { .await; println!("{}", sent.payload); - let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes(), None) + let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes()) .await .unwrap(); assert!(!mime_message.header_exists(HeaderDef::ChatGroupPastMembers)); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 69bb198554..1c7080944f 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -23,6 +23,7 @@ use crate::contact::ContactId; use crate::context::Context; use crate::decrypt::{try_decrypt, validate_detached_signature}; use crate::dehtml::dehtml; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring}; @@ -147,6 +148,23 @@ pub(crate) struct MimeMessage { /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized /// clocks, but not too much. pub(crate) timestamp_sent: i64, + + pub(crate) pre_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PreMessageMode { + /// This is Post-Message + /// it replaces it's Pre-Message attachment if it exists already, + /// and if the Pre-Message does not exist it is treated as normal message + PostMessage, + /// This is a Pre-Message, + /// it adds a message preview for a Post-Message + /// and it is ignored if the Post-Message was downloaded already + PreMessage { + post_msg_rfc724_mid: String, + metadata: Option, + }, } #[derive(Debug, PartialEq)] @@ -240,12 +258,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; impl MimeMessage { /// Parse a mime message. /// - /// If `partial` is set, it contains the full message size in bytes. - pub(crate) async fn from_bytes( - context: &Context, - body: &[u8], - partial: Option, - ) -> Result { + /// This method has some side-effects, + /// such as saving blobs and saving found public keys to the database. + pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result { let mail = mailparse::parse_mail(body)?; let timestamp_rcvd = smeared_time(context); @@ -302,7 +317,7 @@ impl MimeMessage { ); (part, part.ctype.mimetype.parse::()?) } else { - // If it's a partially fetched message, there are no subparts. + // Not a valid signed message, handle it as plaintext. (&mail, mimetype) } } else { @@ -352,6 +367,16 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); + let mut pre_message = if mail + .headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + Some(PreMessageMode::PostMessage) + } else { + None + }; + let mail_raw; // Memory location for a possible decrypted message. let decrypted_msg; // Decrypted signed OpenPGP message. let secrets: Vec = context @@ -580,6 +605,36 @@ impl MimeMessage { signatures.clear(); } + if let (Ok(mail), true) = (mail, is_encrypted) + && let Some(post_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatPostMessageId) + { + let post_msg_rfc724_mid = parse_message_id(&post_msg_rfc724_mid)?; + let metadata = if let Some(value) = mail + .headers + .get_header_value(HeaderDef::ChatPostMessageMetadata) + { + match PreMsgMetadata::try_from_header_value(&value) { + Ok(metadata) => Some(metadata), + Err(error) => { + error!( + context, + "failed to parse metadata header in pre-message: {error:#?}" + ); + None + } + } + } else { + warn!(context, "expected pre-message to have metadata header"); + None + }; + + pre_message = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + metadata, + }); + } + let mut parser = MimeMessage { parts: Vec::new(), headers, @@ -615,33 +670,27 @@ impl MimeMessage { is_bot: None, timestamp_rcvd, timestamp_sent, + pre_message, }; - match partial { - Some(org_bytes) => { - parser - .create_stub_from_partial_download(context, org_bytes) - .await?; + match mail { + Ok(mail) => { + parser.parse_mime_recursive(context, mail, false).await?; + } + Err(err) => { + let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; + + let part = Part { + typ: Viewtype::Text, + msg_raw: Some(txt.to_string()), + msg: txt.to_string(), + // Don't change the error prefix for now, + // receive_imf.rs:lookup_chat_by_reply() checks it. + error: Some(format!("Decrypting failed: {err:#}")), + ..Default::default() + }; + parser.do_add_single_part(part); } - None => match mail { - Ok(mail) => { - parser.parse_mime_recursive(context, mail, false).await?; - } - Err(err) => { - let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; - - let part = Part { - typ: Viewtype::Text, - msg_raw: Some(txt.to_string()), - msg: txt.to_string(), - // Don't change the error prefix for now, - // receive_imf.rs:lookup_chat_by_reply() checks it. - error: Some(format!("Decrypting failed: {err:#}")), - ..Default::default() - }; - parser.do_add_single_part(part); - } - }, }; let is_location_only = parser.location_kml.is_some() && parser.parts.is_empty(); diff --git a/src/mimeparser/mimeparser_tests.rs b/src/mimeparser/mimeparser_tests.rs index 6d198075b9..42edbba7a8 100644 --- a/src/mimeparser/mimeparser_tests.rs +++ b/src/mimeparser/mimeparser_tests.rs @@ -25,58 +25,54 @@ impl AvatarAction { async fn test_mimeparser_fromheader() { let ctx = TestContext::new_alice().await; - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = - MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi", None) - .await - .unwrap(); + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); // although RFC 2047 says, encoded-words shall not appear inside quoted-string, // this combination is used in the wild eg. by MailMate - let mimemsg = MimeMessage::from_bytes( - &ctx, - b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi", - None, - ) - .await - .unwrap(); + let mimemsg = + MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); @@ -86,7 +82,7 @@ async fn test_mimeparser_fromheader() { async fn test_mimeparser_crash() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -98,7 +94,7 @@ async fn test_mimeparser_crash() { async fn test_get_rfc724_mid_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_with_message_id.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -112,7 +108,7 @@ async fn test_get_rfc724_mid_exists() { async fn test_get_rfc724_mid_not_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(mimeparser.get_rfc724_mid(), None); @@ -324,7 +320,7 @@ async fn test_mailparse_0_16_0_panic() { // There should be an error, but no panic. assert!( - MimeMessage::from_bytes(&context.ctx, &raw[..], None) + MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .is_err() ); @@ -341,7 +337,7 @@ async fn test_parse_first_addr() { test1\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -356,7 +352,7 @@ async fn test_get_parent_timestamp() { \n\ Some reply\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -402,7 +398,7 @@ async fn test_mimeparser_with_context() { --==break==--\n\ \n"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -438,26 +434,26 @@ async fn test_mimeparser_with_avatars() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_attach_txt.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.user_avatar, None); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar_deleted.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete)); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); @@ -467,9 +463,7 @@ async fn test_mimeparser_with_avatars() { let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); let raw = String::from_utf8_lossy(raw).to_string(); let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:"); - let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes(), None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes()).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Image); assert_eq!(mimeparser.user_avatar, None); @@ -485,7 +479,7 @@ async fn test_mimeparser_with_videochat() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/videochat_invitation.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.parts[0].param.get(Param::WebrtcRoom), None); @@ -528,7 +522,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\ --==break==--\n\ ;"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -578,7 +572,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -659,7 +653,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --outer--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -706,7 +700,7 @@ Additional-Message-IDs: \n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -753,7 +747,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -797,7 +791,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::File); @@ -839,7 +833,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ----11019878869865180-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("example".to_string())); @@ -903,7 +897,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu --------------779C1631600DF3DB8C02E53A--"#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Test subject".to_string())); @@ -966,7 +960,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ------=_NextPart_000_0003_01D622B3.CA753E60-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1064,7 +1058,7 @@ From: alice Reply "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1096,7 +1090,7 @@ From: alice > Just a quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1130,7 +1124,7 @@ On 2020-10-25, Bob wrote: > A quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Re: top posting".to_string())); @@ -1148,7 +1142,7 @@ On 2020-10-25, Bob wrote: async fn test_attachment_quote() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/quote_attach.eml"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -1166,7 +1160,7 @@ async fn test_attachment_quote() { async fn test_quote_div() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/gmx-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line"); assert_eq!(mimeparser.parts[0].param.get(Param::Quote).unwrap(), "Now?"); } @@ -1176,7 +1170,7 @@ async fn test_allinkl_blockquote() { // all-inkl.com puts quotes into `
`. let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/allinkl-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert!(mimeparser.parts[0].msg.starts_with("It's 1.0.")); assert_eq!( mimeparser.parts[0].param.get(Param::Quote).unwrap(), @@ -1217,7 +1211,7 @@ async fn test_add_subj_to_multimedia_msg() { async fn test_mime_modified_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_plain_unspecified.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1229,7 +1223,7 @@ async fn test_mime_modified_plain() { async fn test_mime_modified_alt_plain_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1241,7 +1235,7 @@ async fn test_mime_modified_alt_plain_html() { async fn test_mime_modified_alt_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1256,7 +1250,7 @@ async fn test_mime_modified_alt_plain() { async fn test_mime_modified_alt_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1268,7 +1262,7 @@ async fn test_mime_modified_alt_html() { async fn test_mime_modified_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1288,7 +1282,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { assert!(long_txt.len() > DC_DESIRED_TEXT_LEN); { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(mimemsg.is_mime_modified); assert!( mimemsg.parts[0].msg.matches("just repeated").count() @@ -1321,7 +1315,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { t.set_config(Config::Bot, Some("1")).await?; { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(!mimemsg.is_mime_modified); assert_eq!( format!("{}\n", mimemsg.parts[0].msg), @@ -1368,7 +1362,7 @@ async fn test_x_microsoft_original_message_id() { MIME-Version: 1.0\n\ \n\ Does it work with outlook now?\n\ - ", None) + ") .await .unwrap(); assert_eq!( @@ -1418,7 +1412,7 @@ async fn test_extra_imf_headers() -> Result<()> { "Message-ID:", "Chat-Forty-Two: 42\r\nForty-Two: 42\r\nMessage-ID:", ); - let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, payload.as_bytes()).await?; assert!(msg.headers.contains_key("chat-version")); assert!(!msg.headers.contains_key("chat-forty-two")); assert_ne!(msg.headers.contains_key("forty-two"), std_hp_composing); @@ -1582,7 +1576,7 @@ async fn test_ms_exchange_mdn() -> Result<()> { // 1. Test mimeparser directly let mdn = include_bytes!("../../test-data/message/ms_exchange_report_disposition_notification.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn, None).await?; + let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?; assert_eq!(mimeparser.mdn_reports.len(), 1); assert_eq!( mimeparser.mdn_reports[0].original_message_id.as_deref(), @@ -1608,7 +1602,6 @@ async fn test_receive_eml() -> Result<()> { let mime_message = MimeMessage::from_bytes( &alice, include_bytes!("../../test-data/message/attached-eml.eml"), - None, ) .await?; @@ -1651,7 +1644,6 @@ Content-Disposition: reaction\n\ \n\ \u{1F44D}" .as_bytes(), - None, ) .await?; @@ -1673,7 +1665,7 @@ async fn test_jpeg_as_application_octet_stream() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/jpeg-as-application-octet-stream.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1691,7 +1683,7 @@ async fn test_schleuder() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/schleuder.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 2); @@ -1711,7 +1703,7 @@ async fn test_tlsrpt() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/tlsrpt.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1744,7 +1736,6 @@ async fn test_time_in_future() -> Result<()> { Content-Type: text/plain; charset=utf-8\n\ \n\ Hi", - None, ) .await?; @@ -1806,7 +1797,7 @@ Content-Type: text/plain; charset=utf-8 /help "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Some subject".to_string())); @@ -1847,7 +1838,7 @@ async fn test_take_last_header() { Hello\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1900,9 +1891,7 @@ It DOES end with a linebreak.\r \r This is the epilogue. It is also to be ignored."; - let mimeparser = MimeMessage::from_bytes(&context, &raw[..], None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&context, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 2); @@ -1948,7 +1937,7 @@ Message with a correct Message-ID hidden header --luTiGu6GBoVLCvTkzVtmZmwsmhkNMw-- "#; - let message = MimeMessage::from_bytes(t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(t, &raw[..]).await.unwrap(); assert_eq!(message.get_rfc724_mid().unwrap(), "foo@example.org"); } @@ -2126,9 +2115,7 @@ Third alternative. --boundary-- "#; - let message = MimeMessage::from_bytes(context, &raw[..], None) - .await - .unwrap(); + let message = MimeMessage::from_bytes(context, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::Text); assert_eq!(message.parts[0].msg, "Third alternative."); diff --git a/src/param.rs b/src/param.rs index 0640b551ef..01f23c7271 100644 --- a/src/param.rs +++ b/src/param.rs @@ -251,6 +251,13 @@ pub enum Param { /// For info messages: Contact ID in added or removed to a group. ContactAddedRemoved = b'5', + + /// For (pre-)Message: ViewType of the Post-Message, + /// because pre message is always `Viewtype::Text`. + PostMessageViewtype = b'8', + + /// For (pre-)Message: File byte size of Post-Message attachment + PostMessageFileBytes = b'9', } /// An object for handling key=value parameter lists. @@ -441,6 +448,15 @@ impl Params { } self } + + /// Merge in parameters from other Params struct, + /// overwriting the keys that are in both + /// with the values from the new Params struct. + pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self { + let mut new_params = new_params; + self.inner.append(&mut new_params.inner); + self + } } #[cfg(test)] @@ -503,4 +519,18 @@ mod tests { assert_eq!(p.get(Param::Height), Some("14")); Ok(()) } + + #[test] + fn test_merge() -> Result<()> { + let mut p = Params::from_str("w=12\na=5\nh=14")?; + let p2 = Params::from_str("L=1\nh=17")?; + assert_eq!(p.len(), 3); + p.merge_in_from_params(p2); + assert_eq!(p.len(), 4); + assert_eq!(p.get(Param::Width), Some("12")); + assert_eq!(p.get(Param::Height), Some("17")); + assert_eq!(p.get(Param::Forwarded), Some("5")); + assert_eq!(p.get(Param::IsEdited), Some("1")); + Ok(()) + } } diff --git a/src/reaction.rs b/src/reaction.rs index 0a00f7aeb1..82a75ed546 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -392,9 +392,8 @@ mod tests { use crate::chatlist::Chatlist; use crate::config::Config; use crate::contact::{Contact, Origin}; - use crate::download::DownloadState; use crate::message::{MessageState, Viewtype, delete_msgs}; - use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; + use crate::receive_imf::receive_imf; use crate::sql::housekeeping; use crate::test_utils::E2EE_INFO_MSGS; use crate::test_utils::TestContext; @@ -924,73 +923,6 @@ Content-Disposition: reaction\n\ Ok(()) } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_reaction() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - - alice - .create_chat_with_contact("Bob", "bob@example.net") - .await; - - let msg_header = "From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - let msg_full = format!("{msg_header}\n\n100k text..."); - - // Alice downloads message from Bob partially. - let alice_received_message = receive_imf_from_inbox( - &alice, - "first@example.org", - msg_header.as_bytes(), - false, - Some(100000), - ) - .await? - .unwrap(); - let alice_msg_id = *alice_received_message.msg_ids.first().unwrap(); - - // Bob downloads own message on the other device. - let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false) - .await? - .unwrap(); - let bob_msg_id = *bob_received_message.msg_ids.first().unwrap(); - - // Bob reacts to own message. - send_reaction(&bob, bob_msg_id, "👍").await.unwrap(); - let bob_reaction_msg = bob.pop_sent_msg().await; - - // Alice receives a reaction. - alice.recv_msg_hidden(&bob_reaction_msg).await; - - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Available); - - // Alice downloads full message. - receive_imf_from_inbox( - &alice, - "first@example.org", - msg_full.as_bytes(), - false, - None, - ) - .await?; - - // Check that reaction is still on the message after full download. - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Done); - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_reaction_multidevice() -> Result<()> { let mut tcm = TestContextManager::new(); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 4415a87cdb..bd08865a58 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,16 +20,14 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::DownloadState; +use crate::download::{DownloadState, premessage_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table}; use crate::key::{DcKey, Fingerprint}; use crate::key::{self_fingerprint, self_fingerprint_opt}; -use crate::log::LogExt; -use crate::log::warn; -use crate::logged_debug_assert; +use crate::log::{LogExt as _, warn}; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; @@ -49,6 +47,7 @@ use crate::tools::{ self, buf_compress, normalize_text, remove_subject_prefix, validate_broadcast_secret, }; use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location}; +use crate::{logged_debug_assert, mimeparser}; /// This is the struct that is returned after receiving one email (aka MIME message). /// @@ -159,24 +158,7 @@ pub async fn receive_imf( let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?; let rfc724_mid = crate::imap::prefetch_get_message_id(&mail.headers) .unwrap_or_else(crate::imap::create_message_id); - if let Some(download_limit) = context.download_limit().await? { - let download_limit: usize = download_limit.try_into()?; - if imf_raw.len() > download_limit { - let head = std::str::from_utf8(imf_raw)? - .split("\r\n\r\n") - .next() - .context("No empty line in the message")?; - return receive_imf_from_inbox( - context, - &rfc724_mid, - head.as_bytes(), - seen, - Some(imf_raw.len().try_into()?), - ) - .await; - } - } - receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await + receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen).await } /// Emulates reception of a message from "INBOX". @@ -188,9 +170,8 @@ pub(crate) async fn receive_imf_from_inbox( rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { - receive_imf_inner(context, rfc724_mid, imf_raw, seen, is_partial_download).await + receive_imf_inner(context, rfc724_mid, imf_raw, seen).await } /// Inserts a tombstone into `msgs` table @@ -213,7 +194,6 @@ async fn get_to_and_past_contact_ids( context: &Context, mime_parser: &MimeMessage, chat_assignment: &ChatAssignment, - is_partial_download: Option, parent_message: &Option, incoming_origin: Origin, ) -> Result<(Vec>, Vec>)> { @@ -256,7 +236,7 @@ async fn get_to_and_past_contact_ids( ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id), ChatAssignment::MailingListOrBroadcast => None, ChatAssignment::OneOneChat => { - if is_partial_download.is_none() && !mime_parser.incoming { + if !mime_parser.incoming { parent_message.as_ref().map(|m| m.chat_id) } else { None @@ -486,15 +466,17 @@ async fn get_to_and_past_contact_ids( /// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`. /// If the message is so wrong that we didn't even create a database entry, /// returns `Ok(None)`. -/// -/// If `is_partial_download` is set, it contains the full message size in bytes. pub(crate) async fn receive_imf_inner( context: &Context, rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { + ensure!( + !context + .get_config_bool(Config::SimulateReceiveImfError) + .await? + ); if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { info!( context, @@ -502,16 +484,8 @@ pub(crate) async fn receive_imf_inner( String::from_utf8_lossy(imf_raw), ); } - if is_partial_download.is_none() { - ensure!( - !context - .get_config_bool(Config::FailOnReceivingFullMsg) - .await? - ); - } - let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await - { + let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw).await { Err(err) => { warn!(context, "receive_imf: can't parse MIME: {err:#}."); if rfc724_mid.starts_with(GENERATED_PREFIX) { @@ -544,7 +518,15 @@ pub(crate) async fn receive_imf_inner( // check, if the mail is already in our database. // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); - if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) { + // Post-Message just replace the attachment and mofified Params, not the whole message + // This is done in the `handle_post_message` method. + replace_msg_id = None; + replace_chat_id = None; + } else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + // This code handles the download of old partial download stub messages + // It will be removed after a transitioning period, + // after we have released a few versions with pre-messages replace_msg_id = Some(old_msg_id); replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id) .await? @@ -617,11 +599,7 @@ pub(crate) async fn receive_imf_inner( &mime_parser.from, fingerprint, prevent_rename, - is_partial_download.is_some() - && mime_parser - .get_header(HeaderDef::ContentType) - .unwrap_or_default() - .starts_with("multipart/encrypted"), + false, ) .await? { @@ -653,22 +631,14 @@ pub(crate) async fn receive_imf_inner( .await? .filter(|p| Some(p.id) != replace_msg_id); - let chat_assignment = decide_chat_assignment( - context, - &mime_parser, - &parent_message, - rfc724_mid, - from_id, - &is_partial_download, - ) - .await?; + let chat_assignment = + decide_chat_assignment(context, &mime_parser, &parent_message, rfc724_mid, from_id).await?; info!(context, "Chat assignment is {chat_assignment:?}."); let (to_ids, past_ids) = get_to_and_past_contact_ids( context, &mime_parser, &chat_assignment, - is_partial_download, &parent_message, incoming_origin, ) @@ -775,7 +745,6 @@ pub(crate) async fn receive_imf_inner( to_id, allow_creation, &mut mime_parser, - is_partial_download, parent_message, ) .await?; @@ -791,7 +760,6 @@ pub(crate) async fn receive_imf_inner( rfc724_mid_orig, from_id, seen, - is_partial_download, replace_msg_id, prevent_rename, chat_id, @@ -959,9 +927,7 @@ pub(crate) async fn receive_imf_inner( let delete_server_after = context.get_config_delete_server_after().await?; if !received_msg.msg_ids.is_empty() { - let target = if received_msg.needs_delete_job - || (delete_server_after == Some(0) && is_partial_download.is_none()) - { + let target = if received_msg.needs_delete_job || delete_server_after == Some(0) { Some(context.get_delete_msgs_target().await?) } else { None @@ -990,7 +956,7 @@ pub(crate) async fn receive_imf_inner( } } - if is_partial_download.is_none() && mime_parser.is_call() { + if mime_parser.is_call() { context .handle_call_msg(insert_msg_id, &mime_parser, from_id) .await?; @@ -1039,7 +1005,7 @@ pub(crate) async fn receive_imf_inner( /// * `find_key_contact_by_addr`: if true, we only know the e-mail address /// of the contact, but not the fingerprint, /// yet want to assign the message to some key-contact. -/// This can happen during prefetch or when the message is partially downloaded. +/// This can happen during prefetch. /// If we get it wrong, the message will be placed into the correct /// chat after downloading. /// @@ -1133,7 +1099,6 @@ async fn decide_chat_assignment( parent_message: &Option, rfc724_mid: &str, from_id: ContactId, - is_partial_download: &Option, ) -> Result { let should_trash = if !mime_parser.mdn_reports.is_empty() { info!(context, "Message is an MDN (TRASH)."); @@ -1149,9 +1114,39 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true - } else if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + } else if let Some(pre_message) = &mime_parser.pre_message { + use crate::mimeparser::PreMessageMode::*; + match pre_message { + PostMessage => { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message is a Post-Message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } + PreMessage { + post_msg_rfc724_mid, + .. + } => { + // if post message already exists, then trash/ignore + let post_msg_exists = + premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?; + info!( + context, + "Message is a Pre-Message (post_msg_exists:{post_msg_exists})." + ); + post_msg_exists + } + } + } else if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { info!(context, "Call state changed (TRASH)."); true @@ -1252,7 +1247,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1274,7 +1269,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1316,7 +1311,6 @@ async fn do_chat_assignment( to_id: ContactId, allow_creation: bool, mime_parser: &mut MimeMessage, - is_partial_download: Option, parent_message: Option, ) -> Result<(ChatId, Blocked, bool)> { let is_bot = context.get_config_bool(Config::Bot).await?; @@ -1367,7 +1361,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), create_blocked, from_id, to_ids, @@ -1416,7 +1409,6 @@ async fn do_chat_assignment( to_ids, allow_creation || test_normal_chat.is_some(), create_blocked, - is_partial_download.is_some(), ) .await? { @@ -1498,7 +1490,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), Blocked::Not, from_id, to_ids, @@ -1562,7 +1553,6 @@ async fn do_chat_assignment( to_ids, allow_creation, Blocked::Not, - is_partial_download.is_some(), ) .await? { @@ -1643,7 +1633,6 @@ async fn add_parts( rfc724_mid: &str, from_id: ContactId, seen: bool, - is_partial_download: Option, mut replace_msg_id: Option, prevent_rename: bool, mut chat_id: ChatId, @@ -1715,10 +1704,9 @@ async fn add_parts( .get_rfc724_mid() .unwrap_or(rfc724_mid.to_string()); - // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded. - let mut ephemeral_timer = if is_partial_download.is_some() { - chat_id.get_ephemeral_timer(context).await? - } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) { + // Extract ephemeral timer from the message + let mut ephemeral_timer = if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) + { match value.parse::() { Ok(timer) => timer, Err(err) => { @@ -1921,7 +1909,6 @@ async fn add_parts( let chat_id = if better_msg .as_ref() .is_some_and(|better_msg| better_msg.is_empty()) - && is_partial_download.is_none() { DC_CHAT_ID_TRASH } else { @@ -1970,10 +1957,10 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; + handle_post_message(context, mime_parser, from_id).await?; - if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) { if let Some(call) = @@ -2054,6 +2041,14 @@ async fn add_parts( } }; + if let Some(mimeparser::PreMessageMode::PreMessage { + metadata: Some(metadata), + .. + }) = &mime_parser.pre_message + { + param.apply_from_pre_msg_metadata(metadata); + }; + // If you change which information is skipped if the message is trashed, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified); @@ -2097,14 +2092,20 @@ RETURNING id "#)?; let row_id: MsgId = stmt.query_row(params![ replace_msg_id, - rfc724_mid_orig, + if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message { + post_msg_rfc724_mid + } else { rfc724_mid_orig }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, if trash { ContactId::UNDEFINED } else { from_id }, if trash { ContactId::UNDEFINED } else { to_id }, sort_timestamp, if trash { 0 } else { mime_parser.timestamp_sent }, if trash { 0 } else { mime_parser.timestamp_rcvd }, - if trash { Viewtype::Unknown } else { typ }, + if trash { + Viewtype::Unknown + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + Viewtype::Text + } else { typ }, if trash { MessageState::Undefined } else { state }, if trash { MessengerMessage::No } else { is_dc_message }, if trash || hidden { "" } else { msg }, @@ -2116,7 +2117,11 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { 0 } else { part.bytes as isize }, + if trash { + 0 + } else { + part.bytes as isize + }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { @@ -2130,10 +2135,10 @@ RETURNING id if trash { 0 } else { ephemeral_timestamp }, if trash { DownloadState::Done - } else if is_partial_download.is_some() { - DownloadState::Available } else if mime_parser.decrypting_failed { DownloadState::Undecipherable + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + DownloadState::Available } else { DownloadState::Done }, @@ -2326,6 +2331,82 @@ async fn handle_edit_delete( Ok(()) } +async fn handle_post_message( + context: &Context, + mime_parser: &MimeMessage, + from_id: ContactId, +) -> Result<()> { + if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message { + // if Pre-Message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected Post-Message to have a message id")?; + + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "Download Post-Message: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "Download Post-Message: pre message was not downloaded, yet so treat as normal message" + ); + return Ok(()); + }; + + if original_msg.from_id != from_id { + warn!(context, "Download Post-Message: Bad sender."); + return Ok(()); + } + if let Some(part) = mime_parser.parts.first() { + if !part.typ.has_file() { + warn!( + context, + "Download Post-Message: First mime part's message-viewtype has no file" + ); + return Ok(()); + } + + let edit_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + let mut new_params = original_msg.param.clone(); + new_params + .merge_in_from_params(part.param.clone()) + .remove(Param::PostMessageFileBytes) + .remove(Param::PostMessageViewtype); + context + .sql + .execute( + "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", + ( + new_params.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); + } else { + warn!(context, "Download Post-Message: Not encrypted."); + } + } + } + + Ok(()) +} + async fn tweak_sort_timestamp( context: &Context, mime_parser: &mut MimeMessage, @@ -2415,7 +2496,6 @@ async fn lookup_chat_by_reply( context: &Context, mime_parser: &MimeMessage, parent: &Message, - is_partial_download: &Option, ) -> Result> { // If the message is encrypted and has group ID, // lookup by reply should never be needed @@ -2447,10 +2527,7 @@ async fn lookup_chat_by_reply( } // Do not assign unencrypted messages to encrypted chats. - if is_partial_download.is_none() - && parent_chat.is_encrypted(context).await? - && !mime_parser.was_encrypted() - { + if parent_chat.is_encrypted(context).await? && !mime_parser.was_encrypted() { return Ok(None); } @@ -2467,18 +2544,7 @@ async fn lookup_or_create_adhoc_group( to_ids: &[Option], allow_creation: bool, create_blocked: Blocked, - is_partial_download: bool, ) -> Result> { - // Partial download may be an encrypted message with protected Subject header. We do not want to - // create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable - // messages. Instead, assign the message to 1:1 chat with the sender. - if is_partial_download { - info!( - context, - "Ad-hoc group cannot be created from partial download." - ); - return Ok(None); - } if mime_parser.decrypting_failed { warn!( context, @@ -2614,11 +2680,9 @@ async fn is_probably_private_reply( /// than two members, a new ad hoc group is created. /// /// On success the function returns the created (chat_id, chat_blocked) tuple. -#[expect(clippy::too_many_arguments)] async fn create_group( context: &Context, mime_parser: &mut MimeMessage, - is_partial_download: bool, create_blocked: Blocked, from_id: ContactId, to_ids: &[Option], @@ -2700,7 +2764,7 @@ async fn create_group( if let Some(chat_id) = chat_id { Ok(Some((chat_id, chat_id_blocked))) - } else if is_partial_download || mime_parser.decrypting_failed { + } else if mime_parser.decrypting_failed { // It is possible that the message was sent to a valid, // yet unknown group, which was rejected because // Chat-Group-Name, which is in the encrypted part, was diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index 40a6eddd05..6eafde62b3 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -10,7 +10,6 @@ use crate::chat::{ use crate::chatlist::Chatlist; use crate::constants::DC_GCL_FOR_FORWARDING; use crate::contact; -use crate::download::MIN_DOWNLOAD_LIMIT; use crate::imap::prefetch_should_download; use crate::imex::{ImexMode, imex}; use crate::securejoin::get_securejoin_qr; @@ -19,8 +18,6 @@ use crate::test_utils::{ }; use crate::tools::{SystemTime, time}; -use rand::distr::SampleString; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_outgoing() -> Result<()> { let context = TestContext::new_alice().await; @@ -28,7 +25,7 @@ async fn test_outgoing() -> Result<()> { From: alice@example.org\n\ \n\ hello"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await?; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await?; assert_eq!(mimeparser.incoming, false); Ok(()) } @@ -43,7 +40,7 @@ async fn test_bad_from() { References: \n\ \n\ hello\x00"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -2842,7 +2839,7 @@ References: Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no Message with references."#; - let mime_parser = MimeMessage::from_bytes(&t, &mime[..], None).await?; + let mime_parser = MimeMessage::from_bytes(&t, &mime[..]).await?; let parent = get_parent_message(&t, &mime_parser).await?.unwrap(); assert_eq!(parent.id, first.id); @@ -4417,37 +4414,6 @@ async fn test_adhoc_grp_name_no_prefix() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_download_later() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - let bob = tcm.bob().await; - let bob_chat = bob.create_chat(&alice).await; - - // Generate a random string so OpenPGP does not compress it. - let text = - rand::distr::Alphanumeric.sample_string(&mut rand::rng(), MIN_DOWNLOAD_LIMIT as usize); - - let sent_msg = bob.send_text(bob_chat.id, &text).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - let hi_msg = tcm.send_recv(&bob, &alice, "hi").await; - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); - assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); - - Ok(()) -} - /// Malice can pretend they have the same address as Alice and sends a message encrypted to Alice's /// key but signed with another one. Alice must detect that this message is wrongly signed and not /// treat it as Autocrypt-encrypted. @@ -4482,162 +4448,6 @@ async fn test_outgoing_msg_forgery() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_create_group_with_big_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let ba_contact = bob.add_or_lookup_contact_id(&alice).await; - let ab_chat_id = alice.create_chat(&bob).await.id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - - let bob_grp_id = create_group(&bob, "Group").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - let alice_chat = Chat::load_from_db(&alice, msg.chat_id).await?; - // Incomplete message is assigned to 1:1 chat. - assert_eq!(alice_chat.typ, Chattype::Single); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, alice_chat.id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // Now Bob can send encrypted messages to Alice. - - let bob_grp_id = create_group(&bob, "Group1").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - // Until fully downloaded, an encrypted message must sit in the 1:1 chat. - assert_eq!(msg.chat_id, ab_chat_id); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, ab_chat_id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group1"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // The big message must go away from the 1:1 chat. - let msgs = chat::get_chat_msgs(&alice, ab_chat_id).await?; - assert_eq!(msgs.len(), E2EE_INFO_MSGS); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_group_consistency() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let fiona = tcm.fiona().await; - let bob_id = alice.add_or_lookup_contact_id(&bob).await; - let alice_chat_id = create_group(&alice, "foos").await?; - add_contact_to_chat(&alice, alice_chat_id, bob_id).await?; - - send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?; - let add = alice.pop_sent_msg().await; - bob.recv_msg(&add).await; - let bob_chat_id = bob.get_last_msg().await.chat_id; - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 2); - - // Bob receives partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: , \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - Some(100000), - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // Partial download does not change the member list. - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - // Alice sends normal message to bob, adding fiona. - add_contact_to_chat( - &alice, - alice_chat_id, - alice.add_or_lookup_contact_id(&fiona).await, - ) - .await?; - - bob.recv_msg(&alice.pop_sent_msg().await).await; - - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 3); - - // Bob fully receives the partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: Bob \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - None, - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // After full download, the old message should not change group state. - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_protected_group_add_remove_member_missing_key() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -4876,48 +4686,6 @@ async fn test_references() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_prefer_references_to_downloaded_msgs() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - bob.set_config(Config::DownloadLimit, Some("1")).await?; - let fiona = &tcm.fiona().await; - let alice_bob_id = tcm.send_recv(bob, alice, "hi").await.from_id; - let alice_fiona_id = tcm.send_recv(fiona, alice, "hi").await.from_id; - let alice_chat_id = create_group(alice, "Group").await?; - add_contact_to_chat(alice, alice_chat_id, alice_bob_id).await?; - // W/o fiona the test doesn't work -- the last message is assigned to the 1:1 chat due to - // `is_probably_private_reply()`. - add_contact_to_chat(alice, alice_chat_id, alice_fiona_id).await?; - let sent = alice.send_text(alice_chat_id, "Hi").await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Done); - let bob_chat_id = received.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.gif"); - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let mut sent = alice.send_msg(alice_chat_id, &mut msg).await; - sent.payload = sent - .payload - .replace("References:", "X-Microsoft-Original-References:") - .replace("In-Reply-To:", "X-Microsoft-Original-In-Reply-To:"); - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_ne!(received.chat_id, bob_chat_id); - assert_eq!(received.chat_id, bob.get_chat(alice).await.id); - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let sent = alice.send_msg(alice_chat_id, &mut msg).await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_eq!(received.chat_id, bob_chat_id); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_list_from() -> Result<()> { let t = &TestContext::new_alice().await; @@ -5395,41 +5163,6 @@ async fn test_outgoing_plaintext_two_member_group() -> Result<()> { Ok(()) } -/// Tests that large messages are assigned -/// to non-key-contacts if the type is not `multipart/encrypted`. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_key_contact_lookup() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - - // Create two chats with Alice, both with key-contact and email address contact. - let encrypted_chat = bob.create_chat(alice).await; - let unencrypted_chat = bob.create_email_chat(alice).await; - - let seen = false; - let is_partial_download = Some(9999); - let received = receive_imf_from_inbox( - bob, - "3333@example.org", - b"From: alice@example.org\n\ - To: bob@example.net\n\ - Message-ID: <3333@example.org>\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ - \n\ - hello\n", - seen, - is_partial_download, - ) - .await? - .unwrap(); - - assert_ne!(received.chat_id, encrypted_chat.id); - assert_eq!(received.chat_id, unencrypted_chat.id); - - Ok(()) -} - /// Tests that outgoing unencrypted message /// is assigned to a chat with email-contact. /// diff --git a/src/scheduler.rs b/src/scheduler.rs index b6055d8ea0..0ec2432f80 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -14,13 +14,12 @@ pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; -use crate::download::{DownloadState, download_msg}; +use crate::download::{download_known_post_messages_without_pre_message, download_msgs}; use crate::ephemeral::{self, delete_expired_imap_messages}; use crate::events::EventType; use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, warn}; -use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::stats::maybe_send_stats; @@ -351,38 +350,6 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } -async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let msg_ids = context - .sql - .query_map_vec("SELECT msg_id FROM download", (), |row| { - let msg_id: MsgId = row.get(0)?; - Ok(msg_id) - }) - .await?; - - for msg_id in msg_ids { - if let Err(err) = download_msg(context, msg_id, session).await { - warn!(context, "Failed to download message {msg_id}: {:#}.", err); - - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; - } - context - .sql - .execute("DELETE FROM download WHERE msg_id=?", (msg_id,)) - .await?; - } - - Ok(()) -} - async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, @@ -597,6 +564,11 @@ async fn fetch_idle( delete_expired_imap_messages(ctx) .await .context("delete_expired_imap_messages")?; + + //------- + // TODO: verify that this is the correct position for this call + // in order to guard against lost pre-messages: + download_known_post_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -682,6 +654,7 @@ async fn fetch_idle( Ok(session) } +/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders) async fn simple_imap_loop( ctx: Context, started: oneshot::Sender<()>, diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index dbd4eeb3cb..faf8daadfc 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1499,6 +1499,30 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; .await?; } + inc_and_check(&mut migration_version, 145)?; + if dbversion < migration_version { + // `msg_id` in `download` table is not needed anymore, + // but we still keep it so that it's possible to import a backup into an older DC version, + // because we don't always release at the same time on all platforms. + sql.execute_migration( + "CREATE TABLE download_new ( + rfc724_mid TEXT NOT NULL DEFAULT '', + msg_id INTEGER NOT NULL DEFAULT 0 + ) STRICT; + INSERT OR IGNORE INTO download_new (rfc724_mid, msg_id) + SELECT m.rfc724_mid, d.msg_id FROM download d + JOIN msgs m ON d.msg_id = m.id + WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; + DROP TABLE download; + ALTER TABLE download_new RENAME TO download; + CREATE TABLE available_post_msgs ( + rfc724_mid TEXT NOT NULL + );", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? diff --git a/src/stock_str.rs b/src/stock_str.rs index 5f13d0b4f9..c22257bfd7 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -4,7 +4,6 @@ use std::collections::HashMap; use std::sync::Arc; use anyhow::{Result, bail}; -use humansize::{BINARY, format_size}; use strum::EnumProperty as EnumPropertyTrait; use strum_macros::EnumProperty; use tokio::sync::RwLock; @@ -17,7 +16,6 @@ use crate::contact::{Contact, ContactId}; use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; -use crate::tools::timestamp_to_str; /// Storage for string translations. #[derive(Debug, Clone)] @@ -167,12 +165,6 @@ pub enum StockMessage { ))] QuotaExceedingMsgBody = 98, - #[strum(props(fallback = "%1$s message"))] - PartialDownloadMsgBody = 99, - - #[strum(props(fallback = "Download maximum available until %1$s"))] - DownloadAvailability = 100, - #[strum(props(fallback = "Multi Device Synchronization"))] SyncMsgSubject = 101, @@ -423,6 +415,9 @@ https://delta.chat/donate"))] #[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))] ChatUnencryptedExplanation = 230, + + #[strum(props(fallback = "Contact"))] + Contact = 231, } impl StockMessage { @@ -890,6 +885,11 @@ pub(crate) async fn sticker(context: &Context) -> String { translated(context, StockMessage::Sticker).await } +/// Stock string: `Contact`. +pub(crate) async fn contact(context: &Context) -> String { + translated(context, StockMessage::Contact).await +} + /// Stock string: `Device messages`. pub(crate) async fn device_messages(context: &Context) -> String { translated(context, StockMessage::DeviceMessages).await @@ -1119,21 +1119,6 @@ pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> St .replace("%%", "%") } -/// Stock string: `%1$s message` with placeholder replaced by human-readable size. -pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String { - let size = &format_size(org_bytes, BINARY); - translated(context, StockMessage::PartialDownloadMsgBody) - .await - .replace1(size) -} - -/// Stock string: `Download maximum available until %1$s`. -pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String { - translated(context, StockMessage::DownloadAvailability) - .await - .replace1(×tamp_to_str(timestamp)) -} - /// Stock string: `Incoming Messages`. pub(crate) async fn incoming_messages(context: &Context) -> String { translated(context, StockMessage::IncomingMessages).await @@ -1254,6 +1239,24 @@ pub(crate) async fn chat_unencrypted_explanation(context: &Context) -> String { translated(context, StockMessage::ChatUnencryptedExplanation).await } +impl Viewtype { + /// returns Localized name for message viewtype + pub async fn to_locale_string(&self, context: &Context) -> String { + match self { + Viewtype::Image => image(context).await, + Viewtype::Gif => gif(context).await, + Viewtype::Sticker => sticker(context).await, + Viewtype::Audio => audio(context).await, + Viewtype::Voice => voice_message(context).await, + Viewtype::Video => video(context).await, + Viewtype::File => file(context).await, + Viewtype::Webxdc => "Mini App".to_owned(), + Viewtype::Vcard => contact(context).await, + Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), + } + } +} + impl Context { /// Set the stock string for the [StockMessage]. /// diff --git a/src/stock_str/stock_str_tests.rs b/src/stock_str/stock_str_tests.rs index 8af98be53f..37c03efabf 100644 --- a/src/stock_str/stock_str_tests.rs +++ b/src/stock_str/stock_str_tests.rs @@ -118,14 +118,6 @@ async fn test_quota_exceeding_stock_str() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_msg_body() -> Result<()> { - let t = TestContext::new().await; - let str = partial_download_msg_body(&t, 1024 * 1024).await; - assert_eq!(str, "1 MiB message"); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_device_chats() { let t = TestContext::new_alice().await; diff --git a/src/test_utils.rs b/src/test_utils.rs index fbdd4f2a2b..54b1866e1e 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,6 +10,7 @@ use std::path::Path; use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; +use anyhow::Result; use async_channel::{self as channel, Receiver, Sender}; use chat::ChatItem; use deltachat_contact_tools::{ContactAddress, EmailAddress}; @@ -711,6 +712,32 @@ impl TestContext { }) } + pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec> { + self.ctx + .sql + .query_map_vec( + "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?", + (msg_id,), + |row| { + let _id: MsgId = row.get(0)?; + let msg_id: MsgId = row.get(1)?; + let mime: String = row.get(2)?; + let recipients: String = row.get(3)?; + Ok((msg_id, mime, recipients)) + }, + ) + .await + .unwrap() + .into_iter() + .map(|(msg_id, mime, recipients)| SentMessage { + payload: mime, + sender_msg_id: msg_id, + sender_context: &self.ctx, + recipients, + }) + .collect() + } + /// Parses a message. /// /// Parsing a message does not run the entire receive pipeline, but is not without @@ -719,7 +746,7 @@ impl TestContext { /// unlikely to be affected as the message would be processed again in exactly the /// same way. pub(crate) async fn parse_msg(&self, msg: &SentMessage<'_>) -> MimeMessage { - MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes(), None) + MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes()) .await .unwrap() } @@ -1663,6 +1690,21 @@ Until the false-positive is fixed: } } +/// Method to create a test image file +pub(crate) fn create_test_image(width: u32, height: u32) -> Result> { + use image::{ImageBuffer, Rgb, RgbImage}; + use std::io::Cursor; + + let mut img: RgbImage = ImageBuffer::new(width, height); + // fill with some pattern so it stays large after compression + for (x, y, pixel) in img.enumerate_pixels_mut() { + *pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]); + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?; + Ok(bytes) +} + mod tests { use super::*; diff --git a/src/tests.rs b/src/tests.rs index 6e642dce74..b7ae08fbad 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,3 +1,4 @@ mod account_events; mod aeap; +mod pre_messages; mod verified_chats; diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs new file mode 100644 index 0000000000..a0387ddee4 --- /dev/null +++ b/src/tests/pre_messages.rs @@ -0,0 +1,6 @@ +mod additional_text; +mod forward_and_save; +mod legacy; +mod receiving; +mod sending; +mod util; diff --git a/src/tests/pre_messages/additional_text.rs b/src/tests/pre_messages/additional_text.rs new file mode 100644 index 0000000000..b894d34e98 --- /dev/null +++ b/src/tests/pre_messages/additional_text.rs @@ -0,0 +1,40 @@ +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::message::Viewtype; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; + +/// Test the addition of the download info to message text +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_additional_text_on_different_viewtypes() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let a_group_id = alice.create_group_with_members("test group", &[bob]).await; + + tcm.section("Test metadata preview text for File"); + let (pre_message, _, _) = + send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned()); + + tcm.section("Test metadata preview text for webxdc app"); + let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc)); + assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned()); + + tcm.section("Test metadata preview text for Image"); + + let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/forward_and_save.rs b/src/tests/pre_messages/forward_and_save.rs new file mode 100644 index 0000000000..a3ca16ed8a --- /dev/null +++ b/src/tests/pre_messages/forward_and_save.rs @@ -0,0 +1,122 @@ +//! Tests about forwarding and saving Pre-Messages +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::chat::{self}; +use crate::chat::{forward_msgs, save_msgs}; +use crate::chatlist::get_last_message_for_chat; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::TestContextManager; + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_forwarding_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("forward pre message and check it on bobs side"); + forward_msgs(bob, &[bob_msg.id], bob_msg.chat_id).await?; + let forwarded_msg_id = get_last_message_for_chat(bob, bob_msg.chat_id) + .await? + .unwrap(); + let forwarded_msg = Message::load_from_db(bob, forwarded_msg_id).await?; + assert_eq!(forwarded_msg.is_forwarded(), true); + assert_eq!(forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text); + assert!(forwarded_msg.additional_text.is_empty()); + tcm.section("check it on alices side"); + let sent_forward_msg = bob.pop_sent_msg().await; + let alice_forwarded_msg = alice.recv_msg(&sent_forward_msg).await; + assert!(alice_forwarded_msg.additional_text.is_empty()); + assert_eq!(alice_forwarded_msg.is_forwarded(), true); + assert_eq!(alice_forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + alice_forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + + Ok(()) +} + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_saving_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("save pre message and check it"); + save_msgs(bob, &[bob_msg.id]).await?; + let saved_msg_id = get_last_message_for_chat(bob, bob.get_self_chat().await.id) + .await? + .unwrap(); + let saved_msg = Message::load_from_db(bob, saved_msg_id).await?; + assert!(saved_msg.additional_text.is_empty()); + assert!(saved_msg.get_original_msg_id(bob).await?.is_some()); + assert_eq!(saved_msg.download_state(), DownloadState::Done); + assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/legacy.rs b/src/tests/pre_messages/legacy.rs new file mode 100644 index 0000000000..f8f086d1bd --- /dev/null +++ b/src/tests/pre_messages/legacy.rs @@ -0,0 +1,61 @@ +//! Test that downloading old stub messages still works +use anyhow::Result; + +use crate::download::DownloadState; +use crate::receive_imf::receive_imf_from_inbox; +use crate::test_utils::TestContext; + +// The code for downloading stub messages stays +// during the transition perios to pre-messages +// so people can still download their files shortly after they updated. +// After there are a few release with pre-message rolled out, +// we will remove the ability to download stub messages and replace the following test +// so it checks that it doesn't crash or that the messages are replaced by sth. +// like "download failed/expired, please ask sender to send it again" +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_stub_message() -> Result<()> { + let t = TestContext::new_alice().await; + + let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ + To: alice@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\ + Content-Type: text/plain"; + + t.sql + .execute( + r#"INSERT INTO chats VALUES( + 11001,100,'bob@example.com',0,'',2,'', + replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,''); + "#, + (), + ) + .await?; + t.sql.execute(r#"INSERT INTO msgs VALUES( + 11001,'Mr.12345678901@example.com','',0, + 11001,11001,1,1763151754,10,10,1,0, + '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', + '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); + "#, ()).await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.get_subject(), "foo"); + assert!(msg.get_text().contains("[97.66 KiB message]")); + + receive_imf_from_inbox( + &t, + "Mr.12345678901@example.com", + format!("{header}\n\n100k text...").as_bytes(), + false, + ) + .await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.get_subject(), "foo"); + assert_eq!(msg.get_text(), "100k text..."); + + Ok(()) +} diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs new file mode 100644 index 0000000000..fd9c376c7a --- /dev/null +++ b/src/tests/pre_messages/receiving.rs @@ -0,0 +1,522 @@ +//! Tests about receiving Pre-Messages and Post-Message +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::EventType; +use crate::chat; +use crate::contact; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata, +}; +use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs}; +use crate::mimeparser::MimeMessage; +use crate::param::Param; +use crate::reaction::{get_msg_reactions, send_reaction}; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; +use crate::webxdc::StatusUpdateSerial; + +/// Test that mimeparser can correctly detect and parse pre-messages and Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?; + let parsed_post_message = MimeMessage::from_bytes(bob, post_message.payload.as_bytes()).await?; + + assert_eq!( + parsed_post_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PostMessage) + ); + + assert_eq!( + parsed_pre_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PreMessage { + post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(), + metadata: Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None + }) + }) + ); + + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000)); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::File)); + assert_eq!(msg.get_filename(), Some("test.bin".to_owned())); + + Ok(()) +} + +/// Test receiving the Post-Message after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_and_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert!(msg.param.exists(Param::PostMessageViewtype)); + assert!(msg.param.exists(Param::PostMessageFileBytes)); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.param.exists(Param::PostMessageViewtype), false); + assert_eq!(msg.param.exists(Param::PostMessageFileBytes), false); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test out of order receiving. Post-Message is received & downloaded before pre-message. +/// In that case pre-message shall be trashed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_out_of_order_receiving() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + let _ = bob.recv_msg_trash(&pre_message).await; + Ok(()) +} + +/// Test receiving the Post-Message after receiving an edit after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_then_edit_and_then_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?; + let edit_request = alice.pop_sent_msg().await; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&edit_request).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "new_text".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "new_text".to_owned()); + Ok(()) +} + +/// Process normal message with file attachment (neither post nor pre message) +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_normal_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes( + alice, + "test.bin", + &vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize], + None, + )?; + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 1); + let message = smtp_rows.first().expect("message exists"); + + let msg = bob.recv_msg(message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for image attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_image() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_image_message(alice, alice_group_id).await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Image)); + // recoded image dimensions + assert_eq!(msg.get_filebytes(bob).await?, Some(149632)); + assert_eq!(msg.get_height(), 1280); + assert_eq!(msg.get_width(), 720); + + Ok(()) +} + +/// Test receiving reaction on pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_reaction_on_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + // Bob receives pre-message + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state(), DownloadState::Available); + + // Alice sends reaction to her own message + send_reaction(alice, alice_msg_id, "👍").await?; + + // Bob receives the reaction + bob.recv_msg_hidden(&alice.pop_sent_msg().await).await; + + // Test if Bob sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + // Bob downloads Post-Message + bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, bob_msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + + // Test if Bob still sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + Ok(()) +} + +/// Tests that fully downloading the message +/// works but does not reappear when it was already deleted +/// (as in the Message-ID already exists in the database +/// and is assigned to the trash chat). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_full_download_after_trashed() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_group_id = bob.create_group_with_members("test group", &[alice]).await; + + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + // Download message from Bob partially. + let alice_msg = alice.recv_msg(&pre_message).await; + + // Delete the received message. + // Note that it remains in the database in the trash chat. + delete_msgs(alice, &[alice_msg.id]).await?; + + // Fully download message after deletion. + alice.recv_msg_trash(&post_message).await; + + // The message does not reappear. + let msg = Message::load_from_db_optional(bob, alice_msg.id).await?; + assert!(msg.is_none()); + + Ok(()) +} + +/// Test that webxdc updates are received for pre-messages +/// and available when the Post-Message is downloaded +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + // Alice sends a larger instance and an update + let (pre_message, post_message, alice_sent_instance_msg_id) = + send_large_webxdc_message(alice, alice_group_id).await?; + alice + .send_webxdc_status_update( + alice_sent_instance_msg_id, + r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, + ) + .await?; + alice.flush_status_updates().await?; + let webxdc_update = alice.pop_sent_msg().await; + + // Bob does not download instance but already receives update + let bob_instance = bob.recv_msg(&pre_message).await; + assert_eq!(bob_instance.download_state, DownloadState::Available); + bob.recv_msg_trash(&webxdc_update).await; + + // Bob downloads instance, updates should be assigned correctly + bob.recv_msg_trash(&post_message).await; + + let bob_instance = bob.get_last_msg().await; + assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); + assert_eq!(bob_instance.download_state, DownloadState::Done); + assert_eq!( + bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0)) + .await?, + r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# + ); + let info = bob_instance.get_webxdc_info(bob).await?; + assert_eq!(info.document, "doc"); + assert_eq!(info.summary, "sum"); + + Ok(()) +} + +/// Test mark seen pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_markseen_pre_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Bob sends a large message to Alice"); + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Alice receives a pre-message message from Bob"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!(msg.state, MessageState::InFresh); + + tcm.section("Alice marks the pre-message as read and sends a MDN"); + markseen_msgs(alice, vec![msg.id]).await?; + assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen); + assert_eq!( + alice + .sql + .count("SELECT COUNT(*) FROM smtp_mdns", ()) + .await?, + 1 + ); + + tcm.section("Alice downloads message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!( + msg.state, + MessageState::InSeen, + "The message state mustn't be downgraded to `InFresh`" + ); + + Ok(()) +} + +/// Test that pre-message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_pre_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that Post-Message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_post_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (_pre_message, post_message, _bob_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that message ordering is still correct after downloading +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_later_keeps_message_order() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Bob sends hi to Alice"); + let hi_msg = tcm.send_recv(bob, alice, "hi").await; + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + + tcm.section("Alice downloads Post-Message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); + + Ok(()) +} + +/// Test that ChatlistItemChanged event is emitted when downloading Post-Message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_chatlist_event_on_post_msg_download() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Alice downloads Post-Message and waits for ChatlistItemChanged event "); + alice.evtracker.clear_events(); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + alice + .evtracker + .get_matching(|e| { + e == &EventType::ChatlistItemChanged { + chat_id: Some(msg.chat_id), + } + }) + .await; + + Ok(()) +} diff --git a/src/tests/pre_messages/sending.rs b/src/tests/pre_messages/sending.rs new file mode 100644 index 0000000000..1d04a4a4d8 --- /dev/null +++ b/src/tests/pre_messages/sending.rs @@ -0,0 +1,337 @@ +//! Tests about sending pre-messages +//! - When to send a pre-message and post-message instead of a normal message +//! - Test that sent pre- and post-message contain the right Headers +//! and that they are send in the correct order (pre-message is sent first.) +use anyhow::Result; +use mailparse::MailHeaderMap; +use tokio::fs; + +use crate::chat::{self, create_group, send_msg}; +use crate::config::Config; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::headerdef::{HeaderDef, HeaderDefMap}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::{self, TestContext, TestContextManager}; +/// Tests that Pre-Message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +/// Also test that Pre-Message is sent first, before the Post-Message +/// And that Autocrypt-gossip and selfavatar never go into Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_sending_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // Pre-Message and Post-Message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?; + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + assert!( + pre_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some() + ); + + assert_eq!( + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + Some(format!("<{}>", msg.rfc724_mid)), + "Post-Message should have the rfc message id of the database message" + ); + + assert_ne!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + "message ids of Pre-Message and Post-Message should be different" + ); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert_eq!(decrypted_post_message.decrypting_failed, false); + assert_eq!( + decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId), + false + ); + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert_eq!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .map(String::from), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId) + ); + assert!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::ChatPostMessageId) + .is_none(), + "no Chat-Post-Message-ID header in unprotected headers of Pre-Message" + ); + + Ok(()) +} + +/// Tests that Pre-Message has autocrypt gossip headers and self avatar +/// and Post-Message doesn't have these headers +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + // simulate conditions for sending self avatar + let avatar_src = alice.get_blobdir().join("avatar.png"); + fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?; + alice + .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap())) + .await?; + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .is_some(), + "tested message is not a pre-message, sending order may be broken" + ); + assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0); + assert_ne!(decrypted_pre_message.user_avatar, None); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some(), + "tested message is not a Post-Message, sending order may be broken" + ); + assert_eq!(decrypted_post_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_post_message.user_avatar, None); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_unecrypted_gets_no_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let chat = alice + .create_chat_with_contact("example", "email@example.org") + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1); + let message_bytes = smtp_rows + .first() + .expect("first element exists") + .payload + .as_bytes(); + let message = mailparse::parse_mail(message_bytes)?; + assert!( + message + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + ); + Ok(()) +} + +/// Tests that no pre message is sent for normal message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + "no 'Chat-Is-Post-Message'-header should be present" + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + // test that pre message is not send for large large text + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + Ok(()) +} + +/// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is Post-Message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + Ok(()) +} + +/// Tests that pre message is not send for large webxdc updates +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../../../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) +} diff --git a/src/tests/pre_messages/util.rs b/src/tests/pre_messages/util.rs new file mode 100644 index 0000000000..0d100fed05 --- /dev/null +++ b/src/tests/pre_messages/util.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use async_zip::tokio::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use futures::io::Cursor as FuturesCursor; +use pretty_assertions::assert_eq; +use tokio_util::compat::FuturesAsyncWriteCompatExt; + +use crate::chat::{self, ChatId}; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::message::{Message, MsgId, Viewtype}; +use crate::test_utils::{SentMessage, TestContext, create_test_image}; + +pub async fn send_large_file_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, + view_type: Viewtype, + content: &[u8], +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let mut msg = Message::new(view_type); + let file_name = if view_type == Viewtype::Webxdc { + "test.xdc" + } else { + "test.bin" + }; + msg.set_file_from_bytes(sender, file_name, content, None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?; + let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("Pre-Message exists"); + let post_message = smtp_rows.get(1).expect("Post-Message exists"); + Ok((pre_message.to_owned(), post_message.to_owned(), msg_id)) +} + +pub async fn send_large_webxdc_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let futures_cursor = FuturesCursor::new(Vec::new()); + let mut buffer = futures_cursor.compat_write(); + let mut writer = ZipFileWriter::with_tokio(&mut buffer); + writer + .write_entry_whole( + ZipEntryBuilder::new("index.html".into(), Compression::Stored), + &[0u8; 1_000_000], + ) + .await?; + writer.close().await?; + let big_webxdc_app = buffer.into_inner().into_inner(); + send_large_file_message(sender, target_chat, Viewtype::Webxdc, &big_webxdc_app).await +} + +pub async fn send_large_image_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + send_large_file_message(sender, target_chat, Viewtype::Image, &test_img).await +} diff --git a/src/webxdc/webxdc_tests.rs b/src/webxdc/webxdc_tests.rs index c0d4f87eef..cf72c459dd 100644 --- a/src/webxdc/webxdc_tests.rs +++ b/src/webxdc/webxdc_tests.rs @@ -10,9 +10,8 @@ use crate::chat::{ }; use crate::chatlist::Chatlist; use crate::config::Config; -use crate::download::DownloadState; use crate::ephemeral; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; use crate::tools::{self, SystemTime}; use crate::{message, sql}; @@ -329,69 +328,6 @@ async fn test_webxdc_contact_request() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { - // Alice sends a larger instance and an update - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat = alice.create_chat(&bob).await; - bob.set_config(Config::DownloadLimit, Some("40000")).await?; - let mut alice_instance = create_webxdc_instance( - &alice, - "chess.xdc", - include_bytes!("../../test-data/webxdc/chess.xdc"), - )?; - let sent1 = alice.send_msg(chat.id, &mut alice_instance).await; - let alice_instance = sent1.load_from_db().await; - alice - .send_webxdc_status_update( - alice_instance.id, - r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, - ) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - - // Bob does not download instance but already receives update - receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - Some(70790), - ) - .await?; - let bob_instance = bob.get_last_msg().await; - bob_instance.chat_id.accept(&bob).await?; - bob.recv_msg_trash(&sent2).await; - assert_eq!(bob_instance.download_state, DownloadState::Available); - - // Bob downloads instance, updates should be assigned correctly - let received_msg = receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - None, - ) - .await? - .unwrap(); - assert_eq!(*received_msg.msg_ids.first().unwrap(), bob_instance.id); - let bob_instance = bob.get_last_msg().await; - assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); - assert_eq!(bob_instance.download_state, DownloadState::Done); - assert_eq!( - bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0)) - .await?, - r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# - ); - let info = bob_instance.get_webxdc_info(&bob).await?; - assert_eq!(info.document, "doc"); - assert_eq!(info.summary, "sum"); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_delete_webxdc_instance() -> Result<()> { let t = TestContext::new_alice().await; From 12edcc0a74aeaee1f11cae9c488bf21e670cfa08 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 14 Dec 2025 18:03:10 +0000 Subject: [PATCH 02/39] remove "Contact" stock string again, use emoji like in summaries --- deltachat-ffi/deltachat.h | 5 ----- src/stock_str.rs | 10 +--------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 18ac329092..37e4ceef8b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -7769,11 +7769,6 @@ void dc_event_unref(dc_event_t* event); /// Used as the first info messages in newly created classic email threads. #define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230 -/// "Contact" -/// -/// Used in summaries. -#define DC_STR_CONTACT 231 - /** * @} */ diff --git a/src/stock_str.rs b/src/stock_str.rs index c22257bfd7..aa3b6fe488 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -415,9 +415,6 @@ https://delta.chat/donate"))] #[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))] ChatUnencryptedExplanation = 230, - - #[strum(props(fallback = "Contact"))] - Contact = 231, } impl StockMessage { @@ -885,11 +882,6 @@ pub(crate) async fn sticker(context: &Context) -> String { translated(context, StockMessage::Sticker).await } -/// Stock string: `Contact`. -pub(crate) async fn contact(context: &Context) -> String { - translated(context, StockMessage::Contact).await -} - /// Stock string: `Device messages`. pub(crate) async fn device_messages(context: &Context) -> String { translated(context, StockMessage::DeviceMessages).await @@ -1251,7 +1243,7 @@ impl Viewtype { Viewtype::Video => video(context).await, Viewtype::File => file(context).await, Viewtype::Webxdc => "Mini App".to_owned(), - Viewtype::Vcard => contact(context).await, + Viewtype::Vcard => "👤".to_string(), Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), } } From 4dca0fc669a2ee92153f7edc67911d142d9d1157 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 20:47:56 +0100 Subject: [PATCH 03/39] fix: Add additional_text also to the summary, in order to prevent empty summaries --- src/summary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/summary.rs b/src/summary.rs index 90ddd1e35e..f47d100bb9 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -262,7 +262,7 @@ impl Message { } }; - let text = self.text.clone(); + let text = self.text.clone() + &self.additional_text; let summary = if let Some(type_file) = type_file { if append_text && !text.is_empty() { From 24d4c3ee591612a895e30793988d46dae849cfcd Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 20:58:51 +0100 Subject: [PATCH 04/39] refactor: Rename download_when_normal_starts->download_later --- src/imap.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index b2150d3ba6..73fac6df37 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -616,7 +616,7 @@ impl Imap { let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); let mut available_post_msgs = Vec::::with_capacity(msgs.len()); - let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); + let mut download_later = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; @@ -728,7 +728,7 @@ impl Imap { // whether it fits download size limit if download_limit.is_none_or(|download_limit| size < download_limit) { - download_when_normal_starts.push(message_id.clone()); + download_later.push(message_id.clone()); } } else { info!(context, "{message_id:?} is not a post-message."); @@ -812,7 +812,7 @@ impl Imap { context, "available_post_msgs: {}, download_when_normal_starts: {}", available_post_msgs.len(), - download_when_normal_starts.len() + download_later.len() ); for rfc724_mid in available_post_msgs { context @@ -820,7 +820,7 @@ impl Imap { .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) .await?; } - for rfc724_mid in download_when_normal_starts { + for rfc724_mid in download_later { context .sql .insert( From f449883ea5c64431b2c62c509cc1fd85bc8b6790 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 21:00:16 +0100 Subject: [PATCH 05/39] refactor: use more idiomatic way of creating Vec, and remove premature optimization --- src/imap.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 73fac6df37..eacacafaa3 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -614,9 +614,9 @@ impl Imap { .context("prefetch")?; let read_cnt = msgs.len(); - let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); - let mut available_post_msgs = Vec::::with_capacity(msgs.len()); - let mut download_later = Vec::::with_capacity(msgs.len()); + let mut uids_fetch: Vec = Vec::new(); + let mut available_post_msgs: Vec = Vec::new(); + let mut download_later: Vec = Vec::new(); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; From ea2eee3526fc582adedfbab00d30818986abc097 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:00:30 +0000 Subject: [PATCH 06/39] improve error message --- src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index f49fe301d5..db897cda54 100644 --- a/src/download.rs +++ b/src/download.rs @@ -155,7 +155,7 @@ pub(crate) async fn download_msg( let Some((server_uid, server_folder)) = row else { // No IMAP record found, we don't know the UID and folder. - return Err(anyhow!("Call download_full() again to try over.")); + return Err(anyhow!("IMAP location for {rfc724_mid:?} post-message is unknown")); }; session From 2a08622afad23dadc488bdb18e0fe82d35a873a8 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:35:11 +0000 Subject: [PATCH 07/39] move download_msgs call --- src/scheduler.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/scheduler.rs b/src/scheduler.rs index 0ec2432f80..8873f326b1 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -501,9 +501,6 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } } - download_msgs(ctx, &mut session) - .await - .context("Failed to download messages")?; session .update_metadata(ctx) .await @@ -613,6 +610,10 @@ async fn fetch_idle( .log_err(ctx) .ok(); + download_msgs(ctx, &mut session) + .await + .context("Failed to download messages")?; + connection.connectivity.set_idle(ctx); ctx.emit_event(EventType::ImapInboxIdle); From 4629fa08aa4b97bda5911c7ac27f6a58ffed61c6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:46:11 +0000 Subject: [PATCH 08/39] log request to download --- src/download.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/download.rs b/src/download.rs index db897cda54..825b935e6c 100644 --- a/src/download.rs +++ b/src/download.rs @@ -79,6 +79,7 @@ impl MsgId { } self.update_download_state(context, DownloadState::InProgress) .await?; + info!(context, "Requesting full download of {:?}.", msg.rfc724_mid()); context .sql .execute( From 81c9226ccb6273b26c4b1139b5f65395fad66f55 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 14 Dec 2025 20:22:14 +0000 Subject: [PATCH 09/39] rustfmt --- src/download.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/download.rs b/src/download.rs index 825b935e6c..09e8d8869d 100644 --- a/src/download.rs +++ b/src/download.rs @@ -79,7 +79,11 @@ impl MsgId { } self.update_download_state(context, DownloadState::InProgress) .await?; - info!(context, "Requesting full download of {:?}.", msg.rfc724_mid()); + info!( + context, + "Requesting full download of {:?}.", + msg.rfc724_mid() + ); context .sql .execute( @@ -156,7 +160,9 @@ pub(crate) async fn download_msg( let Some((server_uid, server_folder)) = row else { // No IMAP record found, we don't know the UID and folder. - return Err(anyhow!("IMAP location for {rfc724_mid:?} post-message is unknown")); + return Err(anyhow!( + "IMAP location for {rfc724_mid:?} post-message is unknown" + )); }; session From 1a125dffd81fc7aa160c088cad8befc5f4498486 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 15 Dec 2025 02:17:16 +0000 Subject: [PATCH 10/39] test: add pre-message test --- deltachat-rpc-client/tests/test_something.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index eda0bc1166..8d80ab12a3 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -10,7 +10,7 @@ import pytest from deltachat_rpc_client import EventType, events -from deltachat_rpc_client.const import MessageState +from deltachat_rpc_client.const import MessageState, DownloadState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError @@ -1062,3 +1062,21 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log): assert chat.num_contacts() == 2 assert msg.get_snapshot().chat.num_contacts() == 2 + + +def test_large_message(acfactory) -> None: + """ + Test sending large message without download limit set, + so it is sent with pre-message but downloaded without user interaction. + """ + alice, bob = acfactory.get_online_accounts(2) + + alice_chat_bob = alice.create_chat(bob) + alice_chat_bob.send_message( + "Hello World, this message is bigger than 5 bytes", + file="../test-data/image/screenshot.jpg", + ) + + msg = bob.wait_for_incoming_msg() + snapshot = msg.get_snapshot() + assert snapshot.text == "Hello World, this message is bigger than 5 bytes" From ffb24df9ade78b55e82509469b8fb0d4ba31785c Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 18 Dec 2025 18:28:26 +0100 Subject: [PATCH 11/39] fix: Properly advance uid_next when skipping a post-message Previously, the uid_next wasn't advanced, which didn't create any problems, but it also was inefficient, because another loop was done trying to fetch the message again (and then finally skipping it, because it's already known). --- src/imap.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/imap.rs b/src/imap.rs index eacacafaa3..4a18dc77ce 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -730,6 +730,7 @@ impl Imap { if download_limit.is_none_or(|download_limit| size < download_limit) { download_later.push(message_id.clone()); } + largest_uid_skipped = Some(uid); } else { info!(context, "{message_id:?} is not a post-message."); From af9f019212a69000c17543acfdc82f2d85827b6f Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 27 Dec 2025 05:05:17 -0300 Subject: [PATCH 12/39] restore test_something.py::test_download_limit_chat_assignment The test just passes, it seems there's no need to remove it. --- deltachat-rpc-client/tests/test_something.py | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 8d80ab12a3..f08a366add 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -685,6 +685,40 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None: assert snapshot.show_padlock +@pytest.mark.parametrize("n_accounts", [3, 2]) +def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): + download_limit = 300000 + + alice, *others = acfactory.get_online_accounts(n_accounts) + bob = others[0] + + alice_group = alice.create_group("test group") + for account in others: + chat = account.create_chat(alice) + chat.send_text("Hello Alice!") + assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!" + + contact = alice.create_contact(account) + alice_group.add_contact(contact) + + bob.set_config("download_limit", str(download_limit)) + + alice_group.send_text("hi") + snapshot = bob.wait_for_incoming_msg().get_snapshot() + assert snapshot.text == "hi" + bob_group = snapshot.chat + + path = tmp_path / "large" + path.write_bytes(os.urandom(download_limit + 1)) + + for i in range(10): + logging.info("Sending message %s", i) + alice_group.send_file(str(path)) + snapshot = bob.wait_for_incoming_msg().get_snapshot() + assert snapshot.download_state == DownloadState.AVAILABLE + assert snapshot.chat == bob_group + + def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages From b72a23ee0be0120c592530fa117abf97a226fd70 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 27 Dec 2025 14:37:53 -0300 Subject: [PATCH 13/39] fix: Update message state on full download, recover test_msg_seen_on_imap_when_downloaded --- src/message/message_tests.rs | 31 +++++++++++++++++++++++++++++++ src/receive_imf.rs | 33 +++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index d43d21220e..7302c723a9 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -326,6 +326,37 @@ async fn test_markseen_msgs() -> Result<()> { Ok(()) } +/// Message has been seen on another device when fully downloaded. `state` should be updated. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + alice.set_config(Config::DownloadLimit, Some("1")).await?; + let bob = &tcm.bob().await; + let bob_chat_id = tcm.send_recv_accept(alice, bob, "hi").await.chat_id; + + let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; + let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; + let pre_msg = bob.pop_sent_msg().await; + let msg = alice.recv_msg(&pre_msg).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + + let seen = true; + let rcvd_msg = receive_imf(alice, sent_msg.payload().as_bytes(), seen) + .await? + .unwrap(); + assert_eq!(rcvd_msg.chat_id, DC_CHAT_ID_TRASH); + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert!(msg.get_showpadlock()); + assert_eq!(msg.state, MessageState::InSeen); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_state() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index bd08865a58..7fd3388219 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1957,7 +1957,7 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; - handle_post_message(context, mime_parser, from_id).await?; + handle_post_message(context, mime_parser, from_id, state).await?; if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded @@ -2335,6 +2335,7 @@ async fn handle_post_message( context: &Context, mime_parser: &MimeMessage, from_id: ContactId, + state: MessageState, ) -> Result<()> { if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message { // if Pre-Message exist, replace attachment @@ -2384,19 +2385,23 @@ async fn handle_post_message( .remove(Param::PostMessageFileBytes) .remove(Param::PostMessageViewtype); context - .sql - .execute( - "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", - ( - new_params.to_string(), - part.typ, - part.bytes as isize, - part.error.as_deref().unwrap_or_default(), - DownloadState::Done as u32, - original_msg.id, - ), - ) - .await?; + .sql + .execute( + " +UPDATE msgs SET param=?, type=?, bytes=?, error=?, state=max(state,?), download_state=? +WHERE id=? + ", + ( + new_params.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + state, + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; context.emit_msgs_changed(original_msg.chat_id, original_msg.id); } else { warn!(context, "Download Post-Message: Not encrypted."); From cbc9daad162defc634d328d3e132f7b969a95481 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 18 Dec 2025 16:35:39 +0100 Subject: [PATCH 14/39] fix: Don't ignore post-messages that we have seen referenced by pre-message before There is a bug in pre-messages: - A pre-message adds an entry to the `msgs` table with the `rfc724_mid` of the post-message - If the pre-message and post-message are fetched in separate cycles: - `prefetch_should_download()` returns false, because `rfc724_mid_exists()` returns something - so, the message is not added to `download_later`, and never automatically downloaded. --- deltachat-rpc-client/tests/test_something.py | 2 ++ src/imap.rs | 6 +---- src/message.rs | 26 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index f08a366add..804697ac68 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -1112,5 +1112,7 @@ def test_large_message(acfactory) -> None: ) msg = bob.wait_for_incoming_msg() + msgs_changed_event = bob.wait_for_msgs_changed_event() + assert msg.id == msgs_changed_event.msg_id snapshot = msg.get_snapshot() assert snapshot.text == "Hello World, this message is bigger than 5 bytes" diff --git a/src/imap.rs b/src/imap.rs index 4a18dc77ce..b9ed291023 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -2232,11 +2232,7 @@ pub(crate) async fn prefetch_should_download( message_id: &str, mut flags: impl Iterator>, ) -> Result { - if message::rfc724_mid_exists(context, message_id) - .await? - .is_some() - { - markseen_on_imap_table(context, message_id).await?; + if message::rfc724_mid_download_tried(context, message_id).await? { return Ok(false); } diff --git a/src/message.rs b/src/message.rs index ea29a9b67d..e633a7a916 100644 --- a/src/message.rs +++ b/src/message.rs @@ -2182,6 +2182,32 @@ pub(crate) async fn rfc724_mid_exists_ex( Ok(res) } +/// Returns `true` iff there is a message +/// with the given `rfc724_mid` +/// and a download state other than `DownloadState::Available` +/// (i.e. a download state where it was already tried to download the message). +pub(crate) async fn rfc724_mid_download_tried(context: &Context, rfc724_mid: &str) -> Result { + let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>'); + if rfc724_mid.is_empty() { + warn!( + context, + "Empty rfc724_mid passed to rfc724_mid_download_tried" + ); + return Ok(false); + } + + let res = context + .sql + .exists( + "SELECT COUNT(*) FROM msgs + WHERE rfc724_mid=? AND download_state<>?", + (rfc724_mid, DownloadState::Available), + ) + .await?; + + Ok(res) +} + /// Given a list of Message-IDs, returns the most relevant message found in the database. /// /// Relevance here is `(download_state == Done, index)`, where `index` is an index of Message-ID in From 1754008b9390225343293110b6d0f848bdf61f0b Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 28 Dec 2025 14:27:28 -0300 Subject: [PATCH 15/39] Add a replacement for test_partial_group_consistency checking that pre-messages modify group members There are no partial messages anymore, so i have no idea what else to check. --- src/receive_imf/receive_imf_tests.rs | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index 6eafde62b3..46256c5c0f 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -4448,6 +4448,54 @@ async fn test_outgoing_msg_forgery() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_pre_msg_group_consistency() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let bob_id = alice.add_or_lookup_contact_id(bob).await; + let alice_chat_id = create_group(alice, "foos").await?; + add_contact_to_chat(alice, alice_chat_id, bob_id).await?; + + send_text_msg(alice, alice_chat_id, "populate".to_string()).await?; + let add = alice.pop_sent_msg().await; + bob.recv_msg(&add).await; + let bob_chat_id = bob.get_last_msg().await.chat_id; + bob_chat_id.accept(bob).await?; + let contacts = get_chat_contacts(bob, bob_chat_id).await?; + assert_eq!(contacts.len(), 2); + + add_contact_to_chat( + alice, + alice_chat_id, + alice.add_or_lookup_contact_id(fiona).await, + ) + .await?; + // This message is lost. + alice.pop_sent_msg().await; + + // Pre-message adds the new member. + let file_bytes = include_bytes!("../../test-data/image/screenshot.gif"); + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(alice, "a.jpg", file_bytes, None)?; + let full_msg = alice.send_msg(alice_chat_id, &mut msg).await; + let pre_msg = alice.pop_sent_msg().await; + let msg = bob.recv_msg(&pre_msg).await; + assert_eq!(msg.download_state, DownloadState::Available); + let contacts = get_chat_contacts(bob, bob_chat_id).await?; + assert_eq!(contacts.len(), 3); + + remove_contact_from_chat(bob, bob_chat_id, bob.add_or_lookup_contact_id(fiona).await).await?; + bob.pop_sent_msg().await; + + // Full message doesn't readd the removed member. + bob.recv_msg_trash(&full_msg).await; + let contacts = get_chat_contacts(bob, bob_chat_id).await?; + assert_eq!(contacts.len(), 2); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_protected_group_add_remove_member_missing_key() -> Result<()> { let mut tcm = TestContextManager::new(); From 650246ff2a3c1d73d251e1b0850b3171c0856ec3 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 28 Dec 2025 15:59:04 -0300 Subject: [PATCH 16/39] review comments Also rename pre_message_is_downloaded_for() to msg_is_downloaded_for(). It's also used to check if a post-message is downloaded. --- deltachat-ffi/deltachat.h | 4 +- deltachat-jsonrpc/src/api/types/message.rs | 2 +- src/chat.rs | 40 +++--- src/download.rs | 41 +++--- src/download/pre_msg_metadata.rs | 41 +++--- src/headerdef.rs | 4 +- src/imap.rs | 52 +++---- src/imap/session.rs | 2 +- src/message.rs | 14 +- src/mimefactory.rs | 35 +++-- src/mimeparser.rs | 13 +- src/param.rs | 4 +- src/receive_imf.rs | 159 ++++++++++----------- src/scheduler.rs | 5 +- src/stock_str.rs | 1 + src/tests/pre_messages/receiving.rs | 2 +- 16 files changed, 203 insertions(+), 216 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 37e4ceef8b..b474da4234 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4311,8 +4311,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); /** * Get the size of the file. Returns the size of the file associated with a - * message, if applicable. - * If message is a pre-message, then this returns size of the to be downloaded file. + * message, if applicable. + * If message is a pre-message, then this returns the size of the file to be downloaded. * * Typically, this is used to show the size of document files, e.g. a PDF. * diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 27ba8dca0e..384d6ee212 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -94,7 +94,7 @@ pub struct MessageObject { file_mime: Option, /// The size of the file in bytes, if applicable. - /// If message is a pre-message, then this is the size of the to be downloaded file. + /// If message is a pre-message, then this is the size of the file to be downloaded. file_bytes: u64, file_name: Option, diff --git a/src/chat.rs b/src/chat.rs index 42552c0f4b..b8054e65f8 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2739,17 +2739,17 @@ async fn prepare_send_msg( Ok(row_ids) } -/// Renders the Message or splits it into Post-Message and Pre-Message. +/// Renders the Message or splits it into Pre- and Post-Message. /// /// Pre-Message is a small message with metadata which announces a larger Post-Message. /// Post-Messages are not downloaded in the background. /// -/// If pre-message is not nessesary this returns a normal message instead. +/// If pre-message is not nessesary, this returns `None` as the 0th value. async fn render_mime_message_and_pre_message( context: &Context, msg: &mut Message, mimefactory: MimeFactory, -) -> Result<(RenderedEmail, Option)> { +) -> Result<(Option, RenderedEmail)> { let needs_pre_message = msg.viewtype.has_file() && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages && msg @@ -2761,12 +2761,15 @@ async fn render_mime_message_and_pre_message( if needs_pre_message { info!( context, - "Message is large and will be split into a pre- and a post-message.", + "Message {} is large and will be split into pre- and post-messages.", msg.id, ); let mut mimefactory_post_msg = mimefactory.clone(); mimefactory_post_msg.set_as_post_message(); - let rendered_msg = mimefactory_post_msg.render(context).await?; + let rendered_msg = mimefactory_post_msg + .render(context) + .await + .context("Failed to render post-message")?; let mut mimefactory_pre_msg = mimefactory; mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); @@ -2778,15 +2781,15 @@ async fn render_mime_message_and_pre_message( if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { warn!( context, - "Pre-message for message (MsgId={}) is larger than expected: {}.", + "Pre-message for message {} is larger than expected: {}.", msg.id, rendered_pre_msg.message.len() ); } - Ok((rendered_msg, Some(rendered_pre_msg))) + Ok((Some(rendered_pre_msg), rendered_msg)) } else { - Ok((mimefactory.render(context).await?, None)) + Ok((None, mimefactory.render(context).await?)) } } @@ -2861,7 +2864,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let (rendered_msg, rendered_pre_msg) = + let (rendered_pre_msg, rendered_msg) = match render_mime_message_and_pre_message(context, msg, mimefactory).await { Ok(res) => Ok(res), Err(err) => { @@ -2873,15 +2876,17 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) { info!( context, - "Message Sizes: Pre-Message {}; Post-Message: {}", + "Message {} sizes: pre-message: {}; post-message: {}.", + msg.id, format_size(pre_msg.message.len(), BINARY), - format_size(post_msg.message.len(), BINARY) + format_size(post_msg.message.len(), BINARY), ); } else { info!( context, - "Message will be sent as normal message (no pre- and post message). Size: {}", - format_size(rendered_msg.message.len(), BINARY) + "Message {} will be sent in one shot (no pre- and post-message). Size: {}.", + msg.id, + format_size(rendered_msg.message.len(), BINARY), ); } @@ -2943,7 +2948,6 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - )?; for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); - // send pre-message before actual message if let Some(pre_msg) = &rendered_pre_msg { let row_id = stmt.execute(( &pre_msg.rfc724_mid, @@ -4341,10 +4345,6 @@ pub async fn forward_msgs_2ctx( } if msg.download_state != DownloadState::Done { - // we don't use Message.get_text() here, - // because it may change in future, - // when UI shows this info itself, - // then the additional_text will not be added in get_text anymore. msg.text += &msg.additional_text; } @@ -4428,10 +4428,6 @@ pub(crate) async fn save_copy_in_self_talk( msg.param.remove(Param::PostMessageViewtype); if msg.download_state != DownloadState::Done { - // we don't use Message.get_text() here, - // because it may change in future, - // when UI shows this info itself, - // then the additional_text will not be added in get_text anymore. msg.text += &msg.additional_text; } diff --git a/src/download.rs b/src/download.rs index 09e8d8869d..36f6339016 100644 --- a/src/download.rs +++ b/src/download.rs @@ -22,8 +22,9 @@ pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; /// From this point onward outgoing messages are considered large /// and get a Pre-Message, which announces the Post-Message. -// this is only about sending so we can modify it any time. -// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB) +/// This is only about sending so we can modify it any time. +/// Current value is a bit less than the minimum auto-download setting from the UIs (which is 160 +/// KiB). pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; /// Max size for pre messages. A warning is emitted when this is exceeded. @@ -208,7 +209,7 @@ impl Session { } } -async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { +async fn set_state_to_failure(context: &Context, rfc724_mid: &str) -> Result<()> { if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { // Update download state to failure // so it can be retried. @@ -237,7 +238,7 @@ async fn available_post_msgs_contains_rfc724_mid( .is_some()) } -async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { +async fn delete_from_available_post_msgs(context: &Context, rfc724_mid: &str) -> Result<()> { context .sql .execute( @@ -248,7 +249,7 @@ async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &s Ok(()) } -async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { +async fn delete_from_downloads(context: &Context, rfc724_mid: &str) -> Result<()> { context .sql .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) @@ -256,11 +257,7 @@ async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Resu Ok(()) } -// this is a dedicated method because it is used in multiple places. -pub(crate) async fn premessage_is_downloaded_for( - context: &Context, - rfc724_mid: &str, -) -> Result { +pub(crate) async fn msg_is_downloaded_for(context: &Context, rfc724_mid: &str) -> Result { Ok(message::rfc724_mid_exists(context, rfc724_mid) .await? .is_some()) @@ -278,30 +275,30 @@ pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> R for rfc724_mid in &rfc724_mids { let res = download_msg(context, rfc724_mid.clone(), session).await; if res.is_ok() { - remove_from_download_table(context, rfc724_mid).await?; - remove_from_available_post_msgs_table(context, rfc724_mid).await?; + delete_from_downloads(context, rfc724_mid).await?; + delete_from_available_post_msgs(context, rfc724_mid).await?; } if let Err(err) = res { warn!( context, "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err ); - if !premessage_is_downloaded_for(context, rfc724_mid).await? { + if !msg_is_downloaded_for(context, rfc724_mid).await? { // This is probably a classical email that vanished before we could download it warn!( context, - "{rfc724_mid} is probably a classical email that vanished before we could download it" + "{rfc724_mid} download failed and there is no downloaded pre-message." ); - remove_from_download_table(context, rfc724_mid).await?; + delete_from_downloads(context, rfc724_mid).await?; } else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? { warn!( context, "{rfc724_mid} is in available_post_msgs table but we failed to fetch it, so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime" ); - set_msg_state_to_failed(context, rfc724_mid).await?; - remove_from_download_table(context, rfc724_mid).await?; - remove_from_available_post_msgs_table(context, rfc724_mid).await?; + set_state_to_failure(context, rfc724_mid).await?; + delete_from_downloads(context, rfc724_mid).await?; + delete_from_available_post_msgs(context, rfc724_mid).await?; } else { // leave the message in DownloadState::InProgress; // it will be downloaded once it arrives. @@ -312,8 +309,8 @@ pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> R Ok(()) } -/// Download known post messages without pre_message -/// in order to guard against lost pre-messages: +/// Downloads known post-messages without pre-messages +/// in order to guard against lost pre-messages. pub(crate) async fn download_known_post_messages_without_pre_message( context: &Context, session: &mut Session, @@ -326,14 +323,14 @@ pub(crate) async fn download_known_post_messages_without_pre_message( }) .await?; for rfc724_mid in &rfc724_mids { - if !premessage_is_downloaded_for(context, rfc724_mid).await? { + if !msg_is_downloaded_for(context, rfc724_mid).await? { // Download the Post-Message unconditionally, // because the Pre-Message got lost. // The message may be in the wrong order, // but at least we have it at all. let res = download_msg(context, rfc724_mid.clone(), session).await; if res.is_ok() { - remove_from_available_post_msgs_table(context, rfc724_mid).await?; + delete_from_available_post_msgs(context, rfc724_mid).await?; } if let Err(err) = res { warn!( diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs index cc19b9e1f7..18e8da223a 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/pre_msg_metadata.rs @@ -17,9 +17,9 @@ pub struct PreMsgMetadata { pub(crate) viewtype: Viewtype, /// the original file name pub(crate) filename: String, - /// Dimensions: width and height of image or video + /// Width and height of the image or video #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) dimensions: Option<(i32, i32)>, + pub(crate) wh: Option<(i32, i32)>, /// Duration of audio file or video in milliseconds #[serde(skip_serializing_if = "Option::is_none")] pub(crate) duration: Option, @@ -41,15 +41,18 @@ impl PreMsgMetadata { .get(Param::Filename) .unwrap_or_default() .to_owned(); - let dimensions = { + let wh = { match ( message.param.get_int(Param::Width), message.param.get_int(Param::Height), ) { (None, None) => None, (Some(width), Some(height)) => Some((width, height)), - _ => { - warn!(context, "Message misses either width or height."); + wh => { + warn!( + context, + "Message {} misses width or height: {:?}.", message.id, wh + ); None } } @@ -60,7 +63,7 @@ impl PreMsgMetadata { size, filename, viewtype: message.viewtype, - dimensions, + wh, duration, })) } @@ -76,7 +79,7 @@ impl PreMsgMetadata { impl Params { /// Applies data from pre_msg_metadata to Params - pub(crate) fn apply_from_pre_msg_metadata( + pub(crate) fn apply_pre_msg_metadata( &mut self, pre_msg_metadata: &PreMsgMetadata, ) -> &mut Self { @@ -88,7 +91,7 @@ impl Params { Param::PostMessageViewtype, pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), ); - if let Some((width, height)) = pre_msg_metadata.dimensions { + if let Some((width, height)) = pre_msg_metadata.wh { self.set(Param::Width, width); self.set(Param::Height, height); } @@ -127,7 +130,7 @@ mod tests { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), - dimensions: None, + wh: None, duration: None, }) ); @@ -154,7 +157,7 @@ mod tests { size: 1816098, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), - dimensions: Some((width as i32, height as i32)), + wh: Some((width as i32, height as i32)), duration: None, }) ); @@ -170,7 +173,7 @@ mod tests { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), - dimensions: None, + wh: None, duration: None, } .to_header_value()?, @@ -181,18 +184,18 @@ mod tests { size: 5_342_765, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), - dimensions: Some((1080, 1920)), + wh: Some((1080, 1920)), duration: None, } .to_header_value()?, - "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"wh\":[1080,1920]}" ); assert_eq!( PreMsgMetadata { size: 5_000, viewtype: Viewtype::Audio, filename: "audio-DD-MM-YY.ogg".to_string(), - dimensions: None, + wh: None, duration: Some(152_310), } .to_header_value()?, @@ -208,25 +211,25 @@ mod tests { fn test_deserialize_from_header() -> Result<()> { assert_eq!( serde_json::from_str::( - "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"wh\":null,\"duration\":null}" )?, PreMsgMetadata { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), - dimensions: None, + wh: None, duration: None, } ); assert_eq!( serde_json::from_str::( - "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"wh\":[1080,1920]}" )?, PreMsgMetadata { size: 5_342_765, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), - dimensions: Some((1080, 1920)), + wh: Some((1080, 1920)), duration: None, } ); @@ -238,7 +241,7 @@ mod tests { size: 5_000, viewtype: Viewtype::Audio, filename: "audio-DD-MM-YY.ogg".to_string(), - dimensions: None, + wh: None, duration: Some(152_310), } ); diff --git a/src/headerdef.rs b/src/headerdef.rs index 4baee1c57c..bfed524e32 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,7 +102,7 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, - /// A message with a large attachment is split into two MIME messages: + /// A message with a large attachment is split into two messages: /// A pre-message, which contains everything but the attachment, /// and a Post-Message. /// The Pre-Message gets a `Chat-Post-Message-Id` header @@ -115,7 +115,7 @@ pub enum HeaderDef { /// This message is preceded by a Pre-Message /// and thus this message can be skipped while fetching messages. - /// This is a cleartext / unproteced header. + /// This is an unprotected header. ChatIsPostMessage, /// [Autocrypt](https://autocrypt.org/) header. diff --git a/src/imap.rs b/src/imap.rs index b9ed291023..50ab87cf1f 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -621,15 +621,10 @@ impl Imap { let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; - let download_limit = { - let download_limit: Option = - context.get_config_parsed(Config::DownloadLimit).await?; - if download_limit == Some(0) { - None - } else { - download_limit - } - }; + let download_limit: Option = context + .get_config_parsed(Config::DownloadLimit) + .await? + .filter(|&l| 0 < l); // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { @@ -726,8 +721,7 @@ impl Imap { info!(context, "{message_id:?} is a post-message."); available_post_msgs.push(message_id.clone()); - // whether it fits download size limit - if download_limit.is_none_or(|download_limit| size < download_limit) { + if download_limit.is_none_or(|download_limit| size <= download_limit) { download_later.push(message_id.clone()); } largest_uid_skipped = Some(uid); @@ -811,25 +805,25 @@ impl Imap { if fetch_res.is_ok() { info!( context, - "available_post_msgs: {}, download_when_normal_starts: {}", + "available_post_msgs: {}, download_later: {}.", available_post_msgs.len(), - download_later.len() + download_later.len(), ); - for rfc724_mid in available_post_msgs { - context - .sql - .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) - .await?; - } - for rfc724_mid in download_later { - context - .sql - .insert( - "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)", - (rfc724_mid,), - ) - .await?; - } + let trans_fn = |t: &mut rusqlite::Transaction| { + let mut stmt = t.prepare("INSERT INTO available_post_msgs VALUES (?)")?; + for rfc724_mid in available_post_msgs { + stmt.execute((rfc724_mid,)) + .context("INSERT INTO available_post_msgs")?; + } + let mut stmt = + t.prepare("INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)")?; + for rfc724_mid in download_later { + stmt.execute((rfc724_mid,)) + .context("INSERT INTO download")?; + } + Ok(()) + }; + context.sql.transaction(trans_fn).await?; } // Now fail if fetching failed, so we will @@ -1364,7 +1358,7 @@ impl Session { } for (request_uids, set) in build_sequence_sets(&request_uids)? { - info!(context, "Starting a full FETCH of message set \"{}\".", set); + info!(context, "Starting UID FETCH of message set \"{}\".", set); let mut fetch_responses = self.uid_fetch(&set, BODY_FULL).await.with_context(|| { format!("fetching messages {} from folder \"{}\"", &set, folder) })?; diff --git a/src/imap/session.rs b/src/imap/session.rs index 1d8c2d7110..9d09430861 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,7 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. -/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large +/// - Chat-Is-Post-Message to skip it in background fetch or when it is > `DownloadLimit`. const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ diff --git a/src/message.rs b/src/message.rs index e633a7a916..2a521188a4 100644 --- a/src/message.rs +++ b/src/message.rs @@ -815,6 +815,9 @@ impl Message { } /// Returns the text of the message. + /// + /// Currently this includes `additional_text`, but this may change in future, when the UIs show + /// the necessary info themselves. pub fn get_text(&self) -> String { self.text.clone() + &self.additional_text } @@ -838,9 +841,8 @@ impl Message { } /// Returns the size of the file in bytes, if applicable. - /// If message is a pre-message, then this returns size of the to be downloaded file. + /// If message is a pre-message, then this returns the size of the file to be downloaded. pub async fn get_filebytes(&self, context: &Context) -> Result> { - // if download state is not downloaded then return value from from params metadata if self.download_state != DownloadState::Done && let Some(file_size) = self .param @@ -862,13 +864,11 @@ impl Message { /// then this returns the viewtype it will have when it is downloaded. #[cfg(test)] pub(crate) fn get_post_message_viewtype(&self) -> Option { - if self.download_state != DownloadState::Done - && let Some(viewtype) = self + if self.download_state != DownloadState::Done { + return self .param .get_i64(Param::PostMessageViewtype) - .and_then(Viewtype::from_i64) - { - return Some(viewtype); + .and_then(Viewtype::from_i64); } None } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 049114847a..893e8fa217 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1004,11 +1004,12 @@ impl MimeFactory { )); } else if let Some(PreMessageMode::PreMessage { post_msg_rfc724_mid, - }) = self.pre_message_mode.clone() + }) = self.pre_message_mode.as_ref() { protected_headers.push(( "Chat-Post-Message-ID", - mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(), + mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid.clone()) + .into(), )); } @@ -1869,23 +1870,19 @@ impl MimeFactory { let footer = if is_reaction { "" } else { &self.selfstatus }; - let message_text = if self.pre_message_mode == Some(PreMessageMode::PostMessage) { - "".to_string() - } else { - format!( - "{}{}{}{}{}{}", - fwdhint.unwrap_or_default(), - quoted_text.unwrap_or_default(), - escape_message_footer_marks(final_text), - if !final_text.is_empty() && !footer.is_empty() { - "\r\n\r\n" - } else { - "" - }, - if !footer.is_empty() { "-- \r\n" } else { "" }, - footer - ) - }; + let message_text = format!( + "{}{}{}{}{}{}", + fwdhint.unwrap_or_default(), + quoted_text.unwrap_or_default(), + escape_message_footer_marks(final_text), + if !final_text.is_empty() && !footer.is_empty() { + "\r\n\r\n" + } else { + "" + }, + if !footer.is_empty() { "-- \r\n" } else { "" }, + footer + ); let mut main_part = MimePart::new("text/plain", message_text); if is_reaction { diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 1c7080944f..8590b6bad9 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -154,9 +154,9 @@ pub(crate) struct MimeMessage { #[derive(Debug, Clone, PartialEq)] pub(crate) enum PreMessageMode { - /// This is Post-Message - /// it replaces it's Pre-Message attachment if it exists already, - /// and if the Pre-Message does not exist it is treated as normal message + /// This is a post-message. + /// It replaces its pre-message attachment if it exists already, + /// and if the pre-message does not exist, it is treated as a normal message. PostMessage, /// This is a Pre-Message, /// it adds a message preview for a Post-Message @@ -619,13 +619,16 @@ impl MimeMessage { Err(error) => { error!( context, - "failed to parse metadata header in pre-message: {error:#?}" + "Failed to parse metadata header in pre-message for {post_msg_rfc724_mid}: {error:#}." ); None } } } else { - warn!(context, "expected pre-message to have metadata header"); + warn!( + context, + "Expected pre-message for {post_msg_rfc724_mid} to have metadata header." + ); None }; diff --git a/src/param.rs b/src/param.rs index 01f23c7271..335c54bcd8 100644 --- a/src/param.rs +++ b/src/param.rs @@ -452,7 +452,7 @@ impl Params { /// Merge in parameters from other Params struct, /// overwriting the keys that are in both /// with the values from the new Params struct. - pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self { + pub fn merge_in_params(&mut self, new_params: Self) -> &mut Self { let mut new_params = new_params; self.inner.append(&mut new_params.inner); self @@ -525,7 +525,7 @@ mod tests { let mut p = Params::from_str("w=12\na=5\nh=14")?; let p2 = Params::from_str("L=1\nh=17")?; assert_eq!(p.len(), 3); - p.merge_in_from_params(p2); + p.merge_in_params(p2); assert_eq!(p.len(), 4); assert_eq!(p.get(Param::Width), Some("12")); assert_eq!(p.get(Param::Height), Some("17")); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 7fd3388219..a7532e2496 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,7 +20,7 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::{DownloadState, premessage_is_downloaded_for}; +use crate::download::{DownloadState, msg_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -519,7 +519,7 @@ pub(crate) async fn receive_imf_inner( // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) { - // Post-Message just replace the attachment and mofified Params, not the whole message + // Post-Message just replaces the attachment and modifies Params, not the whole message. // This is done in the `handle_post_message` method. replace_msg_id = None; replace_chat_id = None; @@ -1119,10 +1119,10 @@ async fn decide_chat_assignment( match pre_message { PostMessage => { // if pre message exist, then trash after replacing, otherwise treat as normal message - let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + let pre_message_exists = msg_is_downloaded_for(context, rfc724_mid).await?; info!( context, - "Message is a Post-Message ({}).", + "Message {rfc724_mid} is a post-message ({}).", if pre_message_exists { "pre-message exists already, so trash after replacing attachment" } else { @@ -1136,11 +1136,10 @@ async fn decide_chat_assignment( .. } => { // if post message already exists, then trash/ignore - let post_msg_exists = - premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?; + let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; info!( context, - "Message is a Pre-Message (post_msg_exists:{post_msg_exists})." + "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." ); post_msg_exists } @@ -2046,7 +2045,7 @@ async fn add_parts( .. }) = &mime_parser.pre_message { - param.apply_from_pre_msg_metadata(metadata); + param.apply_pre_msg_metadata(metadata); }; // If you change which information is skipped if the message is trashed, @@ -2117,11 +2116,7 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { - 0 - } else { - part.bytes as isize - }, + if trash { 0 } else { part.bytes as isize }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { @@ -2337,77 +2332,81 @@ async fn handle_post_message( from_id: ContactId, state: MessageState, ) -> Result<()> { - if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message { - // if Pre-Message exist, replace attachment - // only replacing attachment ensures that doesn't overwrite the text if it was edited before. - let rfc724_mid = mime_parser - .get_rfc724_mid() - .context("expected Post-Message to have a message id")?; - - let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { - warn!( - context, - "Download Post-Message: Database entry does not exist." - ); - return Ok(()); - }; - let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { - // else: message is processed like a normal message - warn!( - context, - "Download Post-Message: pre message was not downloaded, yet so treat as normal message" - ); - return Ok(()); - }; + let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message else { + return Ok(()); + }; + // if Pre-Message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected Post-Message to have a message id")?; - if original_msg.from_id != from_id { - warn!(context, "Download Post-Message: Bad sender."); - return Ok(()); - } - if let Some(part) = mime_parser.parts.first() { - if !part.typ.has_file() { - warn!( - context, - "Download Post-Message: First mime part's message-viewtype has no file" - ); - return Ok(()); - } + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "handle_post_message: {rfc724_mid}: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "handle_post_message: {rfc724_mid}: Pre-message was not downloaded yet so treat as normal message." + ); + return Ok(()); + }; + let Some(part) = mime_parser.parts.first() else { + return Ok(()); + }; - let edit_msg_showpadlock = part - .param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or_default(); - - if edit_msg_showpadlock || !original_msg.get_showpadlock() { - let mut new_params = original_msg.param.clone(); - new_params - .merge_in_from_params(part.param.clone()) - .remove(Param::PostMessageFileBytes) - .remove(Param::PostMessageViewtype); - context - .sql - .execute( - " + // Do nothing if safety checks fail, the worst case is the message modifies the chat if the + // sender is a member. + if from_id != original_msg.from_id { + warn!(context, "handle_post_message: {rfc724_mid}: Bad sender."); + return Ok(()); + } + let post_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + if !post_msg_showpadlock && original_msg.get_showpadlock() { + warn!(context, "handle_post_message: {rfc724_mid}: Not encrypted."); + return Ok(()); + } + + if !part.typ.has_file() { + warn!( + context, + "handle_post_message: {rfc724_mid}: First mime part's message-viewtype has no file." + ); + return Ok(()); + } + + let mut new_params = original_msg.param.clone(); + new_params + .merge_in_params(part.param.clone()) + .remove(Param::PostMessageFileBytes) + .remove(Param::PostMessageViewtype); + context + .sql + .execute( + " UPDATE msgs SET param=?, type=?, bytes=?, error=?, state=max(state,?), download_state=? WHERE id=? - ", - ( - new_params.to_string(), - part.typ, - part.bytes as isize, - part.error.as_deref().unwrap_or_default(), - state, - DownloadState::Done as u32, - original_msg.id, - ), - ) - .await?; - context.emit_msgs_changed(original_msg.chat_id, original_msg.id); - } else { - warn!(context, "Download Post-Message: Not encrypted."); - } - } - } + ", + ( + new_params.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + state, + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); Ok(()) } diff --git a/src/scheduler.rs b/src/scheduler.rs index 8873f326b1..51b5b078af 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -562,9 +562,6 @@ async fn fetch_idle( .await .context("delete_expired_imap_messages")?; - //------- - // TODO: verify that this is the correct position for this call - // in order to guard against lost pre-messages: download_known_post_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); @@ -655,7 +652,7 @@ async fn fetch_idle( Ok(session) } -/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders) +/// Simplified IMAP loop to watch non-inbox folders. async fn simple_imap_loop( ctx: Context, started: oneshot::Sender<()>, diff --git a/src/stock_str.rs b/src/stock_str.rs index aa3b6fe488..4a4806d4e4 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -1244,6 +1244,7 @@ impl Viewtype { Viewtype::File => file(context).await, Viewtype::Webxdc => "Mini App".to_owned(), Viewtype::Vcard => "👤".to_string(), + // The following shouldn't normally be shown to users, so translations aren't needed. Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), } } diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index fd9c376c7a..1f2e6e64aa 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -46,7 +46,7 @@ async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), - dimensions: None, + wh: None, duration: None }) }) From b082e75822d9f91f1d00a8234e7b6bee06d91b72 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 30 Dec 2025 03:36:54 -0300 Subject: [PATCH 17/39] test_msg_text_on_lost_pre_msg --- src/tests/pre_messages/receiving.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index 1f2e6e64aa..4f33786261 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -131,6 +131,25 @@ async fn test_out_of_order_receiving() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_msg_text_on_lost_pre_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_chat_id = alice.create_group_with_members("foos", &[bob]).await; + + let file_bytes = include_bytes!("../../../test-data/image/screenshot.gif"); + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(alice, "a.jpg", file_bytes, None)?; + msg.set_text("populate".to_string()); + let full_msg = alice.send_msg(alice_chat_id, &mut msg).await; + let _pre_msg = alice.pop_sent_msg().await; + let msg = bob.recv_msg(&full_msg).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(msg.text, "populate"); + Ok(()) +} + /// Test receiving the Post-Message after receiving an edit after receiving the pre-message /// for file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From eab95b7ae473af0d18604b14d3f7d987d821967b Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 30 Dec 2025 17:51:21 -0300 Subject: [PATCH 18/39] Add test_pre_and_post_msgs_deleted It currently fails because pre-messages aren't deleted on IMAP. --- src/message/message_tests.rs | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index 7302c723a9..a6b4a28be1 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -357,6 +357,43 @@ async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "bug: pre-messages aren't deleted currently"] +async fn test_pre_and_post_msgs_deleted() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_chat_id = alice.create_group_with_members("", &[bob]).await; + + let file_bytes = include_bytes!("../../test-data/image/screenshot.gif"); + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(alice, "a.jpg", file_bytes, None)?; + let full_msg = alice.send_msg(alice_chat_id, &mut msg).await; + let pre_msg = alice.pop_sent_msg().await; + + let rfc724_mid_pre = bob.parse_msg(&pre_msg).await.get_rfc724_mid().unwrap(); + let msg = bob.recv_msg(&pre_msg).await; + assert_ne!(rfc724_mid_pre, msg.rfc724_mid); + bob.recv_msg_trash(&full_msg).await; + for (rfc724_mid, uid) in [(&rfc724_mid_pre, 1), (&msg.rfc724_mid, 2)] { + bob.sql + .execute( + "INSERT INTO imap (transport_id, rfc724_mid, folder, uid, target, uidvalidity) VALUES (1, ?, 'INBOX', ?, 'INBOX', 12345)", + (rfc724_mid, uid), + ) + .await?; + } + + delete_msgs(bob, &[msg.id]).await?; + assert_eq!( + bob.sql + .count("SELECT COUNT(*) FROM imap WHERE target!=''", ()) + .await?, + 0 + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_state() -> Result<()> { let alice = TestContext::new_alice().await; From 801c0c644e6acf4fce7d4be8b82fc8291e496ea6 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 2 Jan 2026 04:25:54 -0300 Subject: [PATCH 19/39] Add test_post_msg_bad_sender --- src/tests/pre_messages/receiving.rs | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index 4f33786261..a4bc25011f 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -150,6 +150,39 @@ async fn test_msg_text_on_lost_pre_msg() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_post_msg_bad_sender() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let chat_id_alice = alice.create_group_with_members("", &[bob, fiona]).await; + let file_bytes = include_bytes!("../../../test-data/image/screenshot.gif"); + + let mut msg_alice = Message::new(Viewtype::Image); + msg_alice.set_file_from_bytes(alice, "a.jpg", file_bytes, None)?; + let post_msg_alice = alice.send_msg(chat_id_alice, &mut msg_alice).await; + let pre_msg_alice = alice.pop_sent_msg().await; + let msg_bob = bob.recv_msg(&pre_msg_alice).await; + assert_eq!(msg_bob.download_state, DownloadState::Available); + + let chat_id_fiona = fiona.recv_msg(&pre_msg_alice).await.chat_id; + chat_id_fiona.accept(fiona).await?; + let mut msg_fiona = Message::new(Viewtype::Image); + msg_fiona.rfc724_mid = msg_alice.rfc724_mid.clone(); + msg_fiona.set_file_from_bytes(fiona, "a.jpg", file_bytes, None)?; + let post_msg_fiona = fiona.send_msg(chat_id_fiona, &mut msg_fiona).await; + let _pre_msg = fiona.pop_sent_msg().await; + bob.recv_msg_trash(&post_msg_fiona).await; + let msg_bob = Message::load_from_db(bob, msg_bob.id).await?; + assert_eq!(msg_bob.download_state, DownloadState::Available); + + bob.recv_msg_trash(&post_msg_alice).await; + let msg_bob = Message::load_from_db(bob, msg_bob.id).await?; + assert_eq!(msg_bob.download_state, DownloadState::Done); + Ok(()) +} + /// Test receiving the Post-Message after receiving an edit after receiving the pre-message /// for file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 0cbfa5d5fb1028db7dc4019dbabc0b4ca66584dd Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 2 Jan 2026 17:41:07 -0300 Subject: [PATCH 20/39] run `ruff check --fix` --- deltachat-rpc-client/tests/test_something.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 804697ac68..2000bfb05b 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -8,9 +8,8 @@ from unittest.mock import MagicMock import pytest - from deltachat_rpc_client import EventType, events -from deltachat_rpc_client.const import MessageState, DownloadState +from deltachat_rpc_client.const import DownloadState, MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError From 33a305714a2b5a3867535c9564d4b0f6db15b737 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 2 Jan 2026 18:49:07 -0300 Subject: [PATCH 21/39] Add test_lost_pre_msg_vs_new_member --- src/tests/pre_messages/receiving.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index a4bc25011f..e478356ca6 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -165,6 +165,7 @@ async fn test_post_msg_bad_sender() -> Result<()> { let pre_msg_alice = alice.pop_sent_msg().await; let msg_bob = bob.recv_msg(&pre_msg_alice).await; assert_eq!(msg_bob.download_state, DownloadState::Available); + let msg_cnt_bob = msg_bob.chat_id.get_msg_cnt(bob).await?; let chat_id_fiona = fiona.recv_msg(&pre_msg_alice).await.chat_id; chat_id_fiona.accept(fiona).await?; @@ -176,6 +177,7 @@ async fn test_post_msg_bad_sender() -> Result<()> { bob.recv_msg_trash(&post_msg_fiona).await; let msg_bob = Message::load_from_db(bob, msg_bob.id).await?; assert_eq!(msg_bob.download_state, DownloadState::Available); + assert_eq!(msg_bob.chat_id.get_msg_cnt(bob).await?, msg_cnt_bob); bob.recv_msg_trash(&post_msg_alice).await; let msg_bob = Message::load_from_db(bob, msg_bob.id).await?; @@ -183,6 +185,31 @@ async fn test_post_msg_bad_sender() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_lost_pre_msg_vs_new_member() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let chat_id_alice = alice.create_group_with_members("", &[bob, fiona]).await; + let file_bytes = include_bytes!("../../../test-data/image/screenshot.gif"); + + let mut msg_alice = Message::new(Viewtype::Image); + msg_alice.set_file_from_bytes(alice, "a.jpg", file_bytes, None)?; + let post_msg_alice = alice.send_msg(chat_id_alice, &mut msg_alice).await; + let _pre_msg = alice.pop_sent_msg().await; + let msg_bob = bob.recv_msg(&post_msg_alice).await; + assert_eq!(msg_bob.download_state, DownloadState::Done); + let chat_id_bob = msg_bob.chat_id; + assert_eq!(chat::get_chat_contacts(bob, chat_id_bob).await?.len(), 3); + + chat_id_bob.accept(bob).await?; + let sent = bob.send_text(chat_id_bob, "Hi all").await; + alice.recv_msg(&sent).await; + fiona.recv_msg_trash(&sent).await; // Undecryptable message + Ok(()) +} + /// Test receiving the Post-Message after receiving an edit after receiving the pre-message /// for file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 0fe055be2c94cacb31dc4e6ea0ec40f184171075 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 05:48:58 -0300 Subject: [PATCH 22/39] fix: Delete pre-messages on IMAP --- src/chat.rs | 18 ++++++++++++++---- src/ephemeral.rs | 6 ++++-- src/message.rs | 19 +++++++++++++------ src/message/message_tests.rs | 1 - src/receive_imf.rs | 7 +++++-- src/sql/migrations.rs | 3 ++- src/tests/pre_messages/legacy.rs | 2 +- 7 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index b8054e65f8..c328292e2b 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -628,8 +628,12 @@ impl ChatId { .sql .transaction(|transaction| { transaction.execute( - "UPDATE imap SET target=? WHERE rfc724_mid IN (SELECT rfc724_mid FROM msgs WHERE chat_id=?)", - (delete_msgs_target, self,), + "UPDATE imap SET target=? WHERE rfc724_mid IN (SELECT rfc724_mid FROM msgs WHERE chat_id=? AND rfc724_mid!='')", + (&delete_msgs_target, self,), + )?; + transaction.execute( + "UPDATE imap SET target=? WHERE rfc724_mid IN (SELECT pre_rfc724_mid FROM msgs WHERE chat_id=? AND pre_rfc724_mid!='')", + (&delete_msgs_target, self,), )?; transaction.execute( "DELETE FROM smtp WHERE msg_id IN (SELECT id FROM msgs WHERE chat_id=?)", @@ -2881,6 +2885,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - format_size(pre_msg.message.len(), BINARY), format_size(post_msg.message.len(), BINARY), ); + msg.pre_rfc724_mid = pre_msg.rfc724_mid.clone(); } else { info!( context, @@ -2927,8 +2932,13 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - context .sql .execute( - "UPDATE msgs SET subject=?, param=? WHERE id=?", - (&msg.subject, msg.param.to_string(), msg.id), + "UPDATE msgs SET pre_rfc724_mid=?, subject=?, param=? WHERE id=?", + ( + &msg.pre_rfc724_mid, + &msg.subject, + msg.param.to_string(), + msg.id, + ), ) .await?; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 087a804b26..352174b254 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -474,8 +474,10 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu // If you change which information is preserved here, also change `MsgId::trash()` // and other places it references. let mut del_msg_stmt = transaction.prepare( - "INSERT OR REPLACE INTO msgs (id, rfc724_mid, timestamp, chat_id) - SELECT ?1, rfc724_mid, timestamp, ? FROM msgs WHERE id=?1", + " +INSERT OR REPLACE INTO msgs (id, rfc724_mid, pre_rfc724_mid, timestamp, chat_id) +SELECT ?1, rfc724_mid, pre_rfc724_mid, timestamp, ? FROM msgs WHERE id=?1 + ", )?; let mut del_location_stmt = transaction.prepare("DELETE FROM locations WHERE independent=1 AND id=?")?; diff --git a/src/message.rs b/src/message.rs index 2a521188a4..6442758691 100644 --- a/src/message.rs +++ b/src/message.rs @@ -133,8 +133,10 @@ impl MsgId { // If you change which information is preserved here, also change // `delete_expired_messages()` and which information `receive_imf::add_parts()` // still adds to the db if chat_id is TRASH. - "INSERT OR REPLACE INTO msgs (id, rfc724_mid, timestamp, chat_id, deleted) - SELECT ?1, rfc724_mid, timestamp, ?, ? FROM msgs WHERE id=?1", + " +INSERT OR REPLACE INTO msgs (id, rfc724_mid, pre_rfc724_mid, timestamp, chat_id, deleted) +SELECT ?1, rfc724_mid, pre_rfc724_mid, timestamp, ?, ? FROM msgs WHERE id=?1 + ", (self, DC_CHAT_ID_TRASH, on_server), ) .await?; @@ -445,6 +447,8 @@ pub struct Message { /// `Message-ID` header value. pub(crate) rfc724_mid: String, + /// `Message-ID` header value of the pre-message, if any. + pub(crate) pre_rfc724_mid: String, /// `In-Reply-To` header value. pub(crate) in_reply_to: Option, @@ -502,6 +506,7 @@ impl Message { "SELECT", " m.id AS id,", " rfc724_mid AS rfc724mid,", + " pre_rfc724_mid AS pre_rfc724mid,", " m.mime_in_reply_to AS mime_in_reply_to,", " m.chat_id AS chat_id,", " m.from_id AS from_id,", @@ -557,6 +562,7 @@ impl Message { let msg = Message { id: row.get("id")?, rfc724_mid: row.get::<_, String>("rfc724mid")?, + pre_rfc724_mid: row.get::<_, String>("pre_rfc724mid")?, in_reply_to: row .get::<_, Option>("mime_in_reply_to")? .and_then(|in_reply_to| parse_message_id(&in_reply_to).ok()), @@ -1751,10 +1757,11 @@ pub async fn delete_msgs_ex( let target = context.get_delete_msgs_target().await?; let update_db = |trans: &mut rusqlite::Transaction| { - trans.execute( - "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, &msg.rfc724_mid), - )?; + let mut stmt = trans.prepare("UPDATE imap SET target=? WHERE rfc724_mid=?")?; + stmt.execute((&target, &msg.rfc724_mid))?; + if !msg.pre_rfc724_mid.is_empty() { + stmt.execute((&target, &msg.pre_rfc724_mid))?; + } trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; trans.execute( "DELETE FROM download WHERE rfc724_mid=?", diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index a6b4a28be1..f1aa3031f7 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -358,7 +358,6 @@ async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "bug: pre-messages aren't deleted currently"] async fn test_pre_and_post_msgs_deleted() -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index a7532e2496..3301bef657 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2060,7 +2060,7 @@ async fn add_parts( INSERT INTO msgs ( id, - rfc724_mid, chat_id, + rfc724_mid, pre_rfc724_mid, chat_id, from_id, to_id, timestamp, timestamp_sent, timestamp_rcvd, type, state, msgrmsg, txt, txt_normalized, subject, param, hidden, @@ -2070,7 +2070,7 @@ INSERT INTO msgs ) VALUES ( ?, - ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, @@ -2094,6 +2094,9 @@ RETURNING id if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message { post_msg_rfc724_mid } else { rfc724_mid_orig }, + if let Some(mimeparser::PreMessageMode::PreMessage { .. }) = &mime_parser.pre_message { + rfc724_mid_orig + } else { "" }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, if trash { ContactId::UNDEFINED } else { from_id }, if trash { ContactId::UNDEFINED } else { to_id }, diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index faf8daadfc..861a114293 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1517,7 +1517,8 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; ALTER TABLE download_new RENAME TO download; CREATE TABLE available_post_msgs ( rfc724_mid TEXT NOT NULL - );", + ); + ALTER TABLE msgs ADD COLUMN pre_rfc724_mid TEXT DEFAULT '';", migration_version, ) .await?; diff --git a/src/tests/pre_messages/legacy.rs b/src/tests/pre_messages/legacy.rs index f8f086d1bd..fbb13f6f0b 100644 --- a/src/tests/pre_messages/legacy.rs +++ b/src/tests/pre_messages/legacy.rs @@ -38,7 +38,7 @@ async fn test_download_stub_message() -> Result<()> { 11001,'Mr.12345678901@example.com','',0, 11001,11001,1,1763151754,10,10,1,0, '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', - '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); + '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0,''); "#, ()).await?; let msg = t.get_last_msg().await; assert_eq!(msg.download_state(), DownloadState::Available); From e9f4f11df3c3f8bbb9d76d1444def61f3f91af77 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 15:47:38 -0300 Subject: [PATCH 23/39] Make {download,available_post_msgs}.rfc724_mid columns PRIMARY KEY --- src/imap.rs | 8 ++++---- src/sql/migrations.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 50ab87cf1f..7ffd3c61f3 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -810,16 +810,16 @@ impl Imap { download_later.len(), ); let trans_fn = |t: &mut rusqlite::Transaction| { - let mut stmt = t.prepare("INSERT INTO available_post_msgs VALUES (?)")?; + let mut stmt = t.prepare("INSERT OR IGNORE INTO available_post_msgs VALUES (?)")?; for rfc724_mid in available_post_msgs { stmt.execute((rfc724_mid,)) - .context("INSERT INTO available_post_msgs")?; + .context("INSERT OR IGNORE INTO available_post_msgs")?; } let mut stmt = - t.prepare("INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)")?; + t.prepare("INSERT OR IGNORE INTO download (rfc724_mid, msg_id) VALUES (?,0)")?; for rfc724_mid in download_later { stmt.execute((rfc724_mid,)) - .context("INSERT INTO download")?; + .context("INSERT OR IGNORE INTO download")?; } Ok(()) }; diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 861a114293..e195a28753 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1506,18 +1506,18 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; // because we don't always release at the same time on all platforms. sql.execute_migration( "CREATE TABLE download_new ( - rfc724_mid TEXT NOT NULL DEFAULT '', + rfc724_mid TEXT PRIMARY KEY, msg_id INTEGER NOT NULL DEFAULT 0 ) STRICT; INSERT OR IGNORE INTO download_new (rfc724_mid, msg_id) SELECT m.rfc724_mid, d.msg_id FROM download d - JOIN msgs m ON d.msg_id = m.id + LEFT JOIN msgs m ON d.msg_id = m.id WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; DROP TABLE download; ALTER TABLE download_new RENAME TO download; CREATE TABLE available_post_msgs ( - rfc724_mid TEXT NOT NULL - ); + rfc724_mid TEXT PRIMARY KEY + ) STRICT; ALTER TABLE msgs ADD COLUMN pre_rfc724_mid TEXT DEFAULT '';", migration_version, ) From f0125fe039eb9a81a0c46ab03611906790224fc5 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 16:07:38 -0300 Subject: [PATCH 24/39] Replace '-' with en dash for file size separation Hyphen shouldn't be surrounded by spaces in general. --- src/message.rs | 4 ++-- src/tests/pre_messages/additional_text.rs | 6 +++--- src/tests/pre_messages/forward_and_save.rs | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/message.rs b/src/message.rs index 6442758691..1b4faa6d5a 100644 --- a/src/message.rs +++ b/src/message.rs @@ -630,10 +630,10 @@ impl Message { .unwrap_or("?".to_owned()); return match viewtype { - Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")), + Viewtype::File => Ok(format!(" [{file_name} – {file_size}]")), _ => { let translated_viewtype = viewtype.to_locale_string(context).await; - Ok(format!(" [{translated_viewtype} - {file_size}]")) + Ok(format!(" [{translated_viewtype} – {file_size}]")) } }; } diff --git a/src/tests/pre_messages/additional_text.rs b/src/tests/pre_messages/additional_text.rs index b894d34e98..0af33ec8c6 100644 --- a/src/tests/pre_messages/additional_text.rs +++ b/src/tests/pre_messages/additional_text.rs @@ -20,21 +20,21 @@ async fn test_additional_text_on_different_viewtypes() -> Result<()> { send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; let msg = bob.recv_msg(&pre_message).await; assert_eq!(msg.text, "test".to_owned()); - assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned()); + assert_eq!(msg.get_text(), "test [test.bin – 976.56 KiB]".to_owned()); tcm.section("Test metadata preview text for webxdc app"); let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?; let msg = bob.recv_msg(&pre_message).await; assert_eq!(msg.text, "test".to_owned()); assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc)); - assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned()); + assert_eq!(msg.get_text(), "test [Mini App – 976.68 KiB]".to_owned()); tcm.section("Test metadata preview text for Image"); let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?; let msg = bob.recv_msg(&pre_message).await; assert_eq!(msg.text, "test".to_owned()); - assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned()); + assert_eq!(msg.get_text(), "test [Image – 146.12 KiB]".to_owned()); Ok(()) } diff --git a/src/tests/pre_messages/forward_and_save.rs b/src/tests/pre_messages/forward_and_save.rs index a3ca16ed8a..0950fcf6ab 100644 --- a/src/tests/pre_messages/forward_and_save.rs +++ b/src/tests/pre_messages/forward_and_save.rs @@ -54,7 +54,7 @@ async fn test_forwarding_pre_message_empty_text() -> Result<()> { ); assert_eq!( forwarded_msg.get_text(), - " [test.bin - 976.56 KiB]".to_owned() + " [test.bin – 976.56 KiB]".to_owned() ); assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text); assert!(forwarded_msg.additional_text.is_empty()); @@ -80,7 +80,7 @@ async fn test_forwarding_pre_message_empty_text() -> Result<()> { ); assert_eq!( alice_forwarded_msg.get_text(), - " [test.bin - 976.56 KiB]".to_owned() + " [test.bin – 976.56 KiB]".to_owned() ); Ok(()) @@ -116,7 +116,7 @@ async fn test_saving_pre_message_empty_text() -> Result<()> { assert!(saved_msg.additional_text.is_empty()); assert!(saved_msg.get_original_msg_id(bob).await?.is_some()); assert_eq!(saved_msg.download_state(), DownloadState::Done); - assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned()); + assert_eq!(saved_msg.get_text(), " [test.bin – 976.56 KiB]".to_owned()); Ok(()) } From a00efb623cd8a65d2661df8da957049792f4897e Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 17:03:11 -0300 Subject: [PATCH 25/39] Make `PreMessageMode` usable without `Option` --- src/mimefactory.rs | 34 +++++++------- src/mimeparser.rs | 16 ++++--- src/receive_imf.rs | 73 ++++++++++++++--------------- src/tests/pre_messages/receiving.rs | 6 +-- 4 files changed, 65 insertions(+), 64 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 893e8fa217..bac4499596 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -63,10 +63,12 @@ pub enum Loaded { #[derive(Debug, Clone, PartialEq)] pub enum PreMessageMode { /// adds the Chat-Is-Post-Message header in unprotected part - PostMessage, + Post, /// adds the Chat-Post-Message-ID header to protected part /// also adds metadata and explicitly excludes attachment - PreMessage { post_msg_rfc724_mid: String }, + Pre { post_msg_rfc724_mid: String }, + /// Atomic ("normal") message. + None, } /// Helper to construct mime messages. @@ -157,8 +159,8 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, - /// This field is used when this is either a pre-message or a Post-Message. - pre_message_mode: Option, + /// Pre-message / post-message / atomic message. + pre_message_mode: PreMessageMode, } /// Result of rendering a message, ready to be submitted to a send job. @@ -513,7 +515,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, - pre_message_mode: None, + pre_message_mode: PreMessageMode::None, }; Ok(factory) } @@ -562,7 +564,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, - pre_message_mode: None, + pre_message_mode: PreMessageMode::None, }; Ok(res) @@ -795,7 +797,7 @@ impl MimeFactory { let rfc724_mid = match &self.loaded { Loaded::Message { msg, .. } => match &self.pre_message_mode { - Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + PreMessageMode::Pre { .. } => create_outgoing_rfc724_mid(), _ => msg.rfc724_mid.clone(), }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), @@ -997,14 +999,14 @@ impl MimeFactory { mail_builder::headers::raw::Raw::new("1.0").into(), )); - if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + if self.pre_message_mode == PreMessageMode::Post { unprotected_headers.push(( "Chat-Is-Post-Message", mail_builder::headers::raw::Raw::new("1").into(), )); - } else if let Some(PreMessageMode::PreMessage { + } else if let PreMessageMode::Pre { post_msg_rfc724_mid, - }) = self.pre_message_mode.as_ref() + } = &self.pre_message_mode { protected_headers.push(( "Chat-Post-Message-ID", @@ -1154,7 +1156,7 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); - if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + if self.pre_message_mode == PreMessageMode::Post { continue; } @@ -1914,7 +1916,7 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + if let PreMessageMode::Pre { .. } = self.pre_message_mode { let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { bail!("Failed to generate metadata for pre-message") }; @@ -1972,7 +1974,7 @@ impl MimeFactory { } self.attach_selfavatar = - self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage); + self.attach_selfavatar && self.pre_message_mode != PreMessageMode::Post; if self.attach_selfavatar { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { @@ -2048,13 +2050,13 @@ impl MimeFactory { } pub fn set_as_post_message(&mut self) { - self.pre_message_mode = Some(PreMessageMode::PostMessage); + self.pre_message_mode = PreMessageMode::Post; } pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) { - self.pre_message_mode = Some(PreMessageMode::PreMessage { + self.pre_message_mode = PreMessageMode::Pre { post_msg_rfc724_mid: post_message.rfc724_mid.clone(), - }); + }; } } diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 8590b6bad9..ff92298dfa 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -149,7 +149,7 @@ pub(crate) struct MimeMessage { /// clocks, but not too much. pub(crate) timestamp_sent: i64, - pub(crate) pre_message: Option, + pub(crate) pre_message: PreMessageMode, } #[derive(Debug, Clone, PartialEq)] @@ -157,14 +157,16 @@ pub(crate) enum PreMessageMode { /// This is a post-message. /// It replaces its pre-message attachment if it exists already, /// and if the pre-message does not exist, it is treated as a normal message. - PostMessage, + Post, /// This is a Pre-Message, /// it adds a message preview for a Post-Message /// and it is ignored if the Post-Message was downloaded already - PreMessage { + Pre { post_msg_rfc724_mid: String, metadata: Option, }, + /// Atomic ("normal") message. + None, } #[derive(Debug, PartialEq)] @@ -372,9 +374,9 @@ impl MimeMessage { .get_header_value(HeaderDef::ChatIsPostMessage) .is_some() { - Some(PreMessageMode::PostMessage) + PreMessageMode::Post } else { - None + PreMessageMode::None }; let mail_raw; // Memory location for a possible decrypted message. @@ -632,10 +634,10 @@ impl MimeMessage { None }; - pre_message = Some(PreMessageMode::PreMessage { + pre_message = PreMessageMode::Pre { post_msg_rfc724_mid, metadata, - }); + }; } let mut parser = MimeMessage { diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 3301bef657..151ed45a3e 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -31,7 +31,9 @@ use crate::log::{LogExt as _, warn}; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; -use crate::mimeparser::{AvatarAction, GossipedKey, MimeMessage, SystemMessage, parse_message_ids}; +use crate::mimeparser::{ + AvatarAction, GossipedKey, MimeMessage, PreMessageMode, SystemMessage, parse_message_ids, +}; use crate::param::{Param, Params}; use crate::peer_channels::{add_gossip_peer_from_header, insert_topic_stub}; use crate::reaction::{Reaction, set_msg_reaction}; @@ -518,7 +520,7 @@ pub(crate) async fn receive_imf_inner( // check, if the mail is already in our database. // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); - if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) { + if mime_parser.pre_message == mimeparser::PreMessageMode::Post { // Post-Message just replaces the attachment and modifies Params, not the whole message. // This is done in the `handle_post_message` method. replace_msg_id = None; @@ -1114,36 +1116,31 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true - } else if let Some(pre_message) = &mime_parser.pre_message { - use crate::mimeparser::PreMessageMode::*; - match pre_message { - PostMessage => { - // if pre message exist, then trash after replacing, otherwise treat as normal message - let pre_message_exists = msg_is_downloaded_for(context, rfc724_mid).await?; - info!( - context, - "Message {rfc724_mid} is a post-message ({}).", - if pre_message_exists { - "pre-message exists already, so trash after replacing attachment" - } else { - "no pre-message -> Keep" - } - ); - pre_message_exists - } - PreMessage { - post_msg_rfc724_mid, - .. - } => { - // if post message already exists, then trash/ignore - let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; - info!( - context, - "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." - ); - post_msg_exists + } else if mime_parser.pre_message == PreMessageMode::Post { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = msg_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message {rfc724_mid} is a post-message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" } - } + ); + pre_message_exists + } else if let PreMessageMode::Pre { + post_msg_rfc724_mid, + .. + } = &mime_parser.pre_message + { + // if post message already exists, then trash/ignore + let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; + info!( + context, + "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." + ); + post_msg_exists } else if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded { @@ -2040,10 +2037,10 @@ async fn add_parts( } }; - if let Some(mimeparser::PreMessageMode::PreMessage { + if let PreMessageMode::Pre { metadata: Some(metadata), .. - }) = &mime_parser.pre_message + } = &mime_parser.pre_message { param.apply_pre_msg_metadata(metadata); }; @@ -2091,10 +2088,10 @@ RETURNING id "#)?; let row_id: MsgId = stmt.query_row(params![ replace_msg_id, - if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message { + if let PreMessageMode::Pre {post_msg_rfc724_mid, ..} = &mime_parser.pre_message { post_msg_rfc724_mid } else { rfc724_mid_orig }, - if let Some(mimeparser::PreMessageMode::PreMessage { .. }) = &mime_parser.pre_message { + if let PreMessageMode::Pre {..} = &mime_parser.pre_message { rfc724_mid_orig } else { "" }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, @@ -2105,7 +2102,7 @@ RETURNING id if trash { 0 } else { mime_parser.timestamp_rcvd }, if trash { Viewtype::Unknown - } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + } else if let PreMessageMode::Pre {..} = mime_parser.pre_message { Viewtype::Text } else { typ }, if trash { MessageState::Undefined } else { state }, @@ -2135,7 +2132,7 @@ RETURNING id DownloadState::Done } else if mime_parser.decrypting_failed { DownloadState::Undecipherable - } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + } else if let PreMessageMode::Pre {..} = mime_parser.pre_message { DownloadState::Available } else { DownloadState::Done @@ -2335,7 +2332,7 @@ async fn handle_post_message( from_id: ContactId, state: MessageState, ) -> Result<()> { - let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message else { + let PreMessageMode::Post = &mime_parser.pre_message else { return Ok(()); }; // if Pre-Message exist, replace attachment diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index e478356ca6..1d9ff8d755 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -35,12 +35,12 @@ async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { assert_eq!( parsed_post_message.pre_message, - Some(crate::mimeparser::PreMessageMode::PostMessage) + crate::mimeparser::PreMessageMode::Post, ); assert_eq!( parsed_pre_message.pre_message, - Some(crate::mimeparser::PreMessageMode::PreMessage { + crate::mimeparser::PreMessageMode::Pre { post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(), metadata: Some(PreMsgMetadata { size: 1_000_000, @@ -49,7 +49,7 @@ async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { wh: None, duration: None }) - }) + } ); Ok(()) From 6407192e5c67201126cafdcafb3177b03668b127 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 17:14:41 -0300 Subject: [PATCH 26/39] decide_chat_assignment: Don't mix existing checks for should_trash with pre-message ones When inserting pre-message checks in the middle, if they evaluate to false, the remaining checks aren't performed. While this may be correct currently, it's hard to maintain. Better move the pre-message checks into a separate block. --- src/receive_imf.rs | 56 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 151ed45a3e..6e6bb41d3e 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1102,7 +1102,7 @@ async fn decide_chat_assignment( rfc724_mid: &str, from_id: ContactId, ) -> Result { - let should_trash = if !mime_parser.mdn_reports.is_empty() { + let mut should_trash = if !mime_parser.mdn_reports.is_empty() { info!(context, "Message is an MDN (TRASH)."); true } else if mime_parser.delivery_report.is_some() { @@ -1116,31 +1116,6 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true - } else if mime_parser.pre_message == PreMessageMode::Post { - // if pre message exist, then trash after replacing, otherwise treat as normal message - let pre_message_exists = msg_is_downloaded_for(context, rfc724_mid).await?; - info!( - context, - "Message {rfc724_mid} is a post-message ({}).", - if pre_message_exists { - "pre-message exists already, so trash after replacing attachment" - } else { - "no pre-message -> Keep" - } - ); - pre_message_exists - } else if let PreMessageMode::Pre { - post_msg_rfc724_mid, - .. - } = &mime_parser.pre_message - { - // if post message already exists, then trash/ignore - let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; - info!( - context, - "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." - ); - post_msg_exists } else if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded { @@ -1211,6 +1186,35 @@ async fn decide_chat_assignment( false }; + should_trash |= if mime_parser.pre_message == PreMessageMode::Post { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = msg_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message {rfc724_mid} is a post-message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } else if let PreMessageMode::Pre { + post_msg_rfc724_mid, + .. + } = &mime_parser.pre_message + { + // if post message already exists, then trash/ignore + let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; + info!( + context, + "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." + ); + post_msg_exists + } else { + false + }; + // Decide on the type of chat we assign the message to. // // The chat may not exist yet, i.e. there may be From 668d05ccb70f8afb0641513964a48c7b5ebd41d7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 3 Jan 2026 20:43:12 +0000 Subject: [PATCH 27/39] Fix python lint --- deltachat-rpc-client/tests/test_something.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 2000bfb05b..847886a6f4 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock import pytest + from deltachat_rpc_client import EventType, events from deltachat_rpc_client.const import DownloadState, MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS From 300d48ee43a229d6e89a4a8d894c3a09904d1639 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 3 Jan 2026 17:46:05 -0300 Subject: [PATCH 28/39] Move download_msgs() right after download_known_post_messages_without_pre_message() Now it goes before `scan_folders()`. Scanning folders isn't that important, it's even debounced to once per minute. Before, `download_msgs()` even preceded the whole `fetch_idle()`. --- src/scheduler.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/scheduler.rs b/src/scheduler.rs index 51b5b078af..b0a97f53e0 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -563,6 +563,9 @@ async fn fetch_idle( .context("delete_expired_imap_messages")?; download_known_post_messages_without_pre_message(ctx, &mut session).await?; + download_msgs(ctx, &mut session) + .await + .context("download_msgs")?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -607,10 +610,6 @@ async fn fetch_idle( .log_err(ctx) .ok(); - download_msgs(ctx, &mut session) - .await - .context("Failed to download messages")?; - connection.connectivity.set_idle(ctx); ctx.emit_event(EventType::ImapInboxIdle); From be5e403a0afc6ac0ab629d9beed8018ddd4261ca Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 4 Jan 2026 13:39:16 -0300 Subject: [PATCH 29/39] fix: prefetch_should_download: Mark already existing outgoing messages as seen on IMAP --- src/imap.rs | 5 +++++ src/message.rs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 7ffd3c61f3..b05a2ef510 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -2227,6 +2227,11 @@ pub(crate) async fn prefetch_should_download( mut flags: impl Iterator>, ) -> Result { if message::rfc724_mid_download_tried(context, message_id).await? { + if let Some(from) = mimeparser::get_from(headers) + && context.is_self_addr(&from.addr).await? + { + markseen_on_imap_table(context, message_id).await?; + } return Ok(false); } diff --git a/src/message.rs b/src/message.rs index 1b4faa6d5a..037439f43e 100644 --- a/src/message.rs +++ b/src/message.rs @@ -2191,8 +2191,8 @@ pub(crate) async fn rfc724_mid_exists_ex( /// Returns `true` iff there is a message /// with the given `rfc724_mid` -/// and a download state other than `DownloadState::Available` -/// (i.e. a download state where it was already tried to download the message). +/// and a download state other than `DownloadState::Available`, +/// i.e. it was already tried to download the message or it's sent locally. pub(crate) async fn rfc724_mid_download_tried(context: &Context, rfc724_mid: &str) -> Result { let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>'); if rfc724_mid.is_empty() { From f6bb1b1aa252921a9b518e6f3389ea6dea4de049 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 4 Jan 2026 15:35:30 -0300 Subject: [PATCH 30/39] Rename PreMsgMetadata to PostMsgMetadata It's the post-message metadata by its nature and it's even put into the Chat-Post-Message-Metadata header. --- src/download.rs | 3 +- ...e_msg_metadata.rs => post_msg_metadata.rs} | 56 +++++++++---------- src/headerdef.rs | 4 +- src/mimefactory.rs | 4 +- src/mimeparser.rs | 6 +- src/receive_imf.rs | 2 +- src/tests/pre_messages/receiving.rs | 6 +- 7 files changed, 40 insertions(+), 41 deletions(-) rename src/download/{pre_msg_metadata.rs => post_msg_metadata.rs} (84%) diff --git a/src/download.rs b/src/download.rs index 36f6339016..a154d5fcd0 100644 --- a/src/download.rs +++ b/src/download.rs @@ -12,7 +12,8 @@ use crate::log::warn; use crate::message::{self, Message, MsgId, rfc724_mid_exists}; use crate::{EventType, chatlist_events}; -pub(crate) mod pre_msg_metadata; +pub(crate) mod post_msg_metadata; +pub(crate) use post_msg_metadata::PostMsgMetadata; /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), diff --git a/src/download/pre_msg_metadata.rs b/src/download/post_msg_metadata.rs similarity index 84% rename from src/download/pre_msg_metadata.rs rename to src/download/post_msg_metadata.rs index 18e8da223a..ee10640c5b 100644 --- a/src/download/pre_msg_metadata.rs +++ b/src/download/post_msg_metadata.rs @@ -10,7 +10,7 @@ use crate::param::{Param, Params}; /// Metadata contained in Pre-Message that describes the Post-Message. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PreMsgMetadata { +pub struct PostMsgMetadata { /// size of the attachment in bytes pub(crate) size: u64, /// Real viewtype of message @@ -25,8 +25,8 @@ pub struct PreMsgMetadata { pub(crate) duration: Option, } -impl PreMsgMetadata { - // Returns PreMsgMetadata for messages with files and None for messages without file attachment +impl PostMsgMetadata { + /// Returns `PostMsgMetadata` for messages with file attachment and `None` otherwise. pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result> { if !message.viewtype.has_file() { return Ok(None); @@ -78,24 +78,24 @@ impl PreMsgMetadata { } impl Params { - /// Applies data from pre_msg_metadata to Params - pub(crate) fn apply_pre_msg_metadata( + /// Applies data from post_msg_metadata to Params + pub(crate) fn apply_post_msg_metadata( &mut self, - pre_msg_metadata: &PreMsgMetadata, + post_msg_metadata: &PostMsgMetadata, ) -> &mut Self { - self.set(Param::PostMessageFileBytes, pre_msg_metadata.size); - if !pre_msg_metadata.filename.is_empty() { - self.set(Param::Filename, &pre_msg_metadata.filename); + self.set(Param::PostMessageFileBytes, post_msg_metadata.size); + if !post_msg_metadata.filename.is_empty() { + self.set(Param::Filename, &post_msg_metadata.filename); } self.set_i64( Param::PostMessageViewtype, - pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), + post_msg_metadata.viewtype.to_i64().unwrap_or_default(), ); - if let Some((width, height)) = pre_msg_metadata.wh { + if let Some((width, height)) = post_msg_metadata.wh { self.set(Param::Width, width); self.set(Param::Height, height); } - if let Some(duration) = pre_msg_metadata.duration { + if let Some(duration) = post_msg_metadata.duration { self.set(Param::Duration, duration); } @@ -113,7 +113,7 @@ mod tests { test_utils::{TestContextManager, create_test_image}, }; - use super::PreMsgMetadata; + use super::PostMsgMetadata; /// Build from message with file attachment #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -123,10 +123,10 @@ mod tests { let mut file_msg = Message::new(Viewtype::File); file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; - let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?; + let post_msg_metadata = PostMsgMetadata::from_msg(alice, &file_msg).await?; assert_eq!( - pre_mesage_metadata, - Some(PreMsgMetadata { + post_msg_metadata, + Some(PostMsgMetadata { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), @@ -150,10 +150,10 @@ mod tests { // this is usually done while sending, // but we don't send it here, so we need to call it ourself image_msg.try_calc_and_set_dimensions(alice).await?; - let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?; + let post_msg_metadata = PostMsgMetadata::from_msg(alice, &image_msg).await?; assert_eq!( - pre_mesage_metadata, - Some(PreMsgMetadata { + post_msg_metadata, + Some(PostMsgMetadata { size: 1816098, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), @@ -169,7 +169,7 @@ mod tests { #[test] fn test_serialize_to_header() -> Result<()> { assert_eq!( - PreMsgMetadata { + PostMsgMetadata { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), @@ -180,7 +180,7 @@ mod tests { "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}" ); assert_eq!( - PreMsgMetadata { + PostMsgMetadata { size: 5_342_765, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), @@ -191,7 +191,7 @@ mod tests { "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"wh\":[1080,1920]}" ); assert_eq!( - PreMsgMetadata { + PostMsgMetadata { size: 5_000, viewtype: Viewtype::Audio, filename: "audio-DD-MM-YY.ogg".to_string(), @@ -210,10 +210,10 @@ mod tests { #[test] fn test_deserialize_from_header() -> Result<()> { assert_eq!( - serde_json::from_str::( + serde_json::from_str::( "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"wh\":null,\"duration\":null}" )?, - PreMsgMetadata { + PostMsgMetadata { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), @@ -222,10 +222,10 @@ mod tests { } ); assert_eq!( - serde_json::from_str::( + serde_json::from_str::( "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"wh\":[1080,1920]}" )?, - PreMsgMetadata { + PostMsgMetadata { size: 5_342_765, viewtype: Viewtype::Image, filename: "vacation.png".to_string(), @@ -234,10 +234,10 @@ mod tests { } ); assert_eq!( - serde_json::from_str::( + serde_json::from_str::( "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" )?, - PreMsgMetadata { + PostMsgMetadata { size: 5_000, viewtype: Viewtype::Audio, filename: "audio-DD-MM-YY.ogg".to_string(), diff --git a/src/headerdef.rs b/src/headerdef.rs index bfed524e32..7ccae1f979 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -109,8 +109,8 @@ pub enum HeaderDef { /// referencing the Post-Message's rfc724_mid. ChatPostMessageId, - /// Announce Post-Message metadata in a Pre-Message. - /// contains serialized PreMsgMetadata struct + /// Announces Post-Message metadata in a Pre-Message. + /// Contains a serialized `PostMsgMetadata` struct. ChatPostMessageMetadata, /// This message is preceded by a Pre-Message diff --git a/src/mimefactory.rs b/src/mimefactory.rs index bac4499596..1c7b564d3d 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -21,7 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG}; use crate::constants::{Chattype, DC_FROM_HANDSHAKE}; use crate::contact::{Contact, ContactId, Origin}; use crate::context::Context; -use crate::download::pre_msg_metadata::PreMsgMetadata; +use crate::download::PostMsgMetadata; use crate::e2ee::EncryptHelper; use crate::ensure_and_debug_assert; use crate::ephemeral::Timer as EphemeralTimer; @@ -1917,7 +1917,7 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { if let PreMessageMode::Pre { .. } = self.pre_message_mode { - let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { + let Some(metadata) = PostMsgMetadata::from_msg(context, &msg).await? else { bail!("Failed to generate metadata for pre-message") }; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index ff92298dfa..fb66ac9605 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -23,7 +23,7 @@ use crate::contact::ContactId; use crate::context::Context; use crate::decrypt::{try_decrypt, validate_detached_signature}; use crate::dehtml::dehtml; -use crate::download::pre_msg_metadata::PreMsgMetadata; +use crate::download::PostMsgMetadata; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring}; @@ -163,7 +163,7 @@ pub(crate) enum PreMessageMode { /// and it is ignored if the Post-Message was downloaded already Pre { post_msg_rfc724_mid: String, - metadata: Option, + metadata: Option, }, /// Atomic ("normal") message. None, @@ -616,7 +616,7 @@ impl MimeMessage { .headers .get_header_value(HeaderDef::ChatPostMessageMetadata) { - match PreMsgMetadata::try_from_header_value(&value) { + match PostMsgMetadata::try_from_header_value(&value) { Ok(metadata) => Some(metadata), Err(error) => { error!( diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 6e6bb41d3e..1dca00798c 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2046,7 +2046,7 @@ async fn add_parts( .. } = &mime_parser.pre_message { - param.apply_pre_msg_metadata(metadata); + param.apply_post_msg_metadata(metadata); }; // If you change which information is skipped if the message is trashed, diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs index 1d9ff8d755..b3ba6a4e51 100644 --- a/src/tests/pre_messages/receiving.rs +++ b/src/tests/pre_messages/receiving.rs @@ -5,9 +5,7 @@ use pretty_assertions::assert_eq; use crate::EventType; use crate::chat; use crate::contact; -use crate::download::{ - DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata, -}; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PostMsgMetadata}; use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs}; use crate::mimeparser::MimeMessage; use crate::param::Param; @@ -42,7 +40,7 @@ async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { parsed_pre_message.pre_message, crate::mimeparser::PreMessageMode::Pre { post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(), - metadata: Some(PreMsgMetadata { + metadata: Some(PostMsgMetadata { size: 1_000_000, viewtype: Viewtype::File, filename: "test.bin".to_string(), From 96a075716705d2c1bbbc85adec3dab5c383400ac Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 4 Jan 2026 16:33:45 -0300 Subject: [PATCH 31/39] Improve download_limit docs --- deltachat-ffi/deltachat.h | 13 ++++++++----- src/config.rs | 10 ++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index b474da4234..4f367cc74b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -486,11 +486,14 @@ char* dc_get_blobdir (const dc_context_t* context); * This is a developer option used for testing polling used as an IDLE fallback. * - `download_limit` = Messages up to this number of bytes are downloaded automatically. * For messages with large attachments, two messages are sent: - * a Pre-Message containing metadata and a Post-Message containing the attachment. - * Pre-Messages are always downloaded and show a placeholder message. - * These messages can be downloaded fully using dc_download_full_msg() later. - * Post-Messages are automatically downloaded if they are smaller than the download_limit. - * 0=no limit (default). + * a Pre-Message containing metadata and text and a Post-Message additionally + * containing the attachment. NB: Some "extra" metadata like avatars and gossiped + * encryption keys is stripped from post-messages to save traffic. + * Pre-Messages are shown as placeholder messages. They can be downloaded fully + * using dc_download_full_msg() later. Post-Messages are automatically + * downloaded if they are smaller than the download_limit. Other messages are + * always auto-downloaded. + * 0 = no limit (default). * Changes affect future messages only. * - `protect_autocrypt` = Enable Header Protection for Autocrypt header. * This is an experimental option not compatible to other MUAs diff --git a/src/config.rs b/src/config.rs index 350e99ea35..0a589738ed 100644 --- a/src/config.rs +++ b/src/config.rs @@ -354,7 +354,17 @@ pub enum Config { DonationRequestNextCheck, /// Defines the max. size (in bytes) of messages downloaded automatically. + /// + /// For messages with large attachments, two messages are sent: + /// a Pre-Message containing metadata and text and a Post-Message additionally + /// containing the attachment. NB: Some "extra" metadata like avatars and gossiped + /// encryption keys is stripped from post-messages to save traffic. + /// Pre-Messages are shown as placeholder messages. They can be downloaded fully using + /// `MsgId::download_full()` later. Post-Messages are automatically downloaded if they are + /// smaller than the download_limit. Other messages are always auto-downloaded. + /// /// 0 = no limit. + /// Changes only affect future messages. #[strum(props(default = "0"))] DownloadLimit, From 5599fa2504d3a2baed08f090598f1ba240344ce8 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sun, 4 Jan 2026 17:26:52 -0300 Subject: [PATCH 32/39] test_download_limit_chat_assignment: Work around lost and reordered pre-messages --- deltachat-rpc-client/tests/test_something.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 847886a6f4..e92804e810 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -711,11 +711,17 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): path = tmp_path / "large" path.write_bytes(os.urandom(download_limit + 1)) + n_done = 0 for i in range(10): logging.info("Sending message %s", i) alice_group.send_file(str(path)) snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.download_state == DownloadState.AVAILABLE + if snapshot.download_state == DownloadState.DONE: + n_done += 1 + # Work around lost and reordered pre-messages. + assert n_done <= 1 + else: + assert snapshot.download_state == DownloadState.AVAILABLE assert snapshot.chat == bob_group From b41127e7415cc50ac29620d5aafeb5e34c51aec7 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 5 Jan 2026 14:27:31 -0300 Subject: [PATCH 33/39] Download small messages first This shouldn't cause message reordering issues because the UIs don't allow to set too small DownloadLimit, but this will help download small, e.g. chat messages first and only then big emails. Even if we decide to revert this suggestion, the test is still useful. --- deltachat-rpc-client/tests/test_something.py | 23 ++++++++++++++++++++ src/imap.rs | 10 ++++++--- src/receive_imf.rs | 2 ++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index e92804e810..5d811e9936 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -725,6 +725,29 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): assert snapshot.chat == bob_group +def test_download_small_msg_first(acfactory, tmp_path): + download_limit = 70000 + + alice, bob0 = acfactory.get_online_accounts(2) + bob1 = bob0.clone() + bob1.set_config("download_limit", str(download_limit)) + + chat = alice.create_chat(bob0) + path = tmp_path / "large_enough" + path.write_bytes(os.urandom(download_limit + 1)) + # Less than 140K, so sent w/o a pre-message. + chat.send_file(str(path)) + chat.send_text("hi") + bob0.create_chat(alice) + assert bob0.wait_for_incoming_msg().get_snapshot().text == "" + assert bob0.wait_for_incoming_msg().get_snapshot().text == "hi" + + bob1.start_io() + bob1.create_chat(alice) + assert bob1.wait_for_incoming_msg().get_snapshot().text == "hi" + assert bob1.wait_for_incoming_msg().get_snapshot().text == "" + + def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages diff --git a/src/imap.rs b/src/imap.rs index b05a2ef510..60c8056921 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -727,9 +727,13 @@ impl Imap { largest_uid_skipped = Some(uid); } else { info!(context, "{message_id:?} is not a post-message."); - - uids_fetch.push(uid); - uid_message_ids.insert(uid, message_id); + if download_limit.is_none_or(|download_limit| size <= download_limit) { + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + } else { + download_later.push(message_id.clone()); + largest_uid_skipped = Some(uid); + } }; } else { largest_uid_skipped = Some(uid); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 1dca00798c..8b954ab8e4 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2392,6 +2392,8 @@ async fn handle_post_message( .merge_in_params(part.param.clone()) .remove(Param::PostMessageFileBytes) .remove(Param::PostMessageViewtype); + // Don't update `chat_id`: even if it differs from pre-message's one somehow so the result + // depends on message download order, we don't want messages jumping across chats. context .sql .execute( From d56d9404b1c76cad07ac524082015c296550de7c Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 5 Jan 2026 15:47:18 -0300 Subject: [PATCH 34/39] test_delete_available_msg --- deltachat-rpc-client/tests/test_something.py | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 5d811e9936..bf53f725ea 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -748,6 +748,31 @@ def test_download_small_msg_first(acfactory, tmp_path): assert bob1.wait_for_incoming_msg().get_snapshot().text == "" +def test_delete_available_msg(acfactory, tmp_path, direct_imap): + # Min. UI setting as of v2.35 + download_limit = 163840 + + alice, bob = acfactory.get_online_accounts(2) + bob.set_config("download_limit", str(download_limit)) + + chat = alice.create_chat(bob) + path = tmp_path / "large" + path.write_bytes(os.urandom(download_limit + 1)) + chat.send_file(str(path)) + bob_msg = bob.wait_for_incoming_msg() + assert bob_msg.get_snapshot().download_state == DownloadState.AVAILABLE + + bob.delete_messages([bob_msg]) + bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) + bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) + while True: + event = bob.wait_for_event() + if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: + break + bob_direct_imap = direct_imap(bob) + assert len(bob_direct_imap.get_all_messages()) == 0 + + def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages From 21f668011489c0ff9f89607ebd408c3bd5926111 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 5 Jan 2026 17:26:34 -0300 Subject: [PATCH 35/39] fix deletion of reordered pre- and post-messages --- src/message/message_tests.rs | 22 ++++++++++++++++++++-- src/receive_imf.rs | 13 +++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index f1aa3031f7..dc63f86a1c 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -359,6 +359,17 @@ async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_pre_and_post_msgs_deleted() -> Result<()> { + let reorder = false; + test_pre_and_post_msgs_deleted_ex(reorder).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_reordered_pre_and_post_msgs_deleted() -> Result<()> { + let reorder = true; + test_pre_and_post_msgs_deleted_ex(reorder).await +} + +async fn test_pre_and_post_msgs_deleted_ex(reorder: bool) -> Result<()> { let mut tcm = TestContextManager::new(); let alice = &tcm.alice().await; let bob = &tcm.bob().await; @@ -371,9 +382,16 @@ async fn test_pre_and_post_msgs_deleted() -> Result<()> { let pre_msg = alice.pop_sent_msg().await; let rfc724_mid_pre = bob.parse_msg(&pre_msg).await.get_rfc724_mid().unwrap(); - let msg = bob.recv_msg(&pre_msg).await; + let msg = if reorder { + let msg = bob.recv_msg(&full_msg).await; + bob.recv_msg_trash(&pre_msg).await; + Message::load_from_db(bob, msg.id).await? + } else { + let msg = bob.recv_msg(&pre_msg).await; + bob.recv_msg_trash(&full_msg).await; + msg + }; assert_ne!(rfc724_mid_pre, msg.rfc724_mid); - bob.recv_msg_trash(&full_msg).await; for (rfc724_mid, uid) in [(&rfc724_mid_pre, 1), (&msg.rfc724_mid, 2)] { bob.sql .execute( diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 8b954ab8e4..effa98aed3 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1204,8 +1204,17 @@ async fn decide_chat_assignment( .. } = &mime_parser.pre_message { - // if post message already exists, then trash/ignore - let post_msg_exists = msg_is_downloaded_for(context, post_msg_rfc724_mid).await?; + let msg_id = rfc724_mid_exists(context, post_msg_rfc724_mid).await?; + if let Some(msg_id) = msg_id { + context + .sql + .execute( + "UPDATE msgs SET pre_rfc724_mid=? WHERE id=?", + (rfc724_mid, msg_id), + ) + .await?; + } + let post_msg_exists = msg_id.is_some(); info!( context, "Message {rfc724_mid} is a pre-message for {post_msg_rfc724_mid} (post_msg_exists:{post_msg_exists})." From 449ac1972aed78f69a1ddcf6881d66d9aed9e27a Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 6 Jan 2026 16:57:48 -0300 Subject: [PATCH 36/39] fix pre-message deletion on the sender side --- deltachat-rpc-client/tests/test_something.py | 27 +++++++++++++------- src/message.rs | 2 +- src/sql/migrations.rs | 3 ++- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index bf53f725ea..70e7cd54f6 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -753,24 +753,33 @@ def test_delete_available_msg(acfactory, tmp_path, direct_imap): download_limit = 163840 alice, bob = acfactory.get_online_accounts(2) + alice.set_config("bcc_self", "1") bob.set_config("download_limit", str(download_limit)) chat = alice.create_chat(bob) path = tmp_path / "large" path.write_bytes(os.urandom(download_limit + 1)) - chat.send_file(str(path)) + alice_msg = chat.send_file(str(path)) bob_msg = bob.wait_for_incoming_msg() assert bob_msg.get_snapshot().download_state == DownloadState.AVAILABLE bob.delete_messages([bob_msg]) - bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) - bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) - while True: - event = bob.wait_for_event() - if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: - break - bob_direct_imap = direct_imap(bob) - assert len(bob_direct_imap.get_all_messages()) == 0 + alice.wait_for_event(EventType.SMTP_MESSAGE_SENT) + alice.wait_for_event(EventType.SMTP_MESSAGE_SENT) + # Avoid DeleteMessages sync message + alice.set_config("bcc_self", "0") + alice.delete_messages([alice_msg]) + for acc in [bob, alice]: + acc.wait_for_event(EventType.MSG_DELETED) + # Messages may be deleted separately + while True: + acc.wait_for_event(EventType.IMAP_MESSAGE_DELETED) + while True: + event = acc.wait_for_event() + if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: + break + if len(direct_imap(acc).get_all_messages()) == 0: + break def test_markseen_contact_request(acfactory): diff --git a/src/message.rs b/src/message.rs index 037439f43e..d8c971cc17 100644 --- a/src/message.rs +++ b/src/message.rs @@ -2174,7 +2174,7 @@ pub(crate) async fn rfc724_mid_exists_ex( .query_row_optional( &("SELECT id, timestamp_sent, MIN(".to_string() + expr - + ") FROM msgs WHERE rfc724_mid=? + + ") FROM msgs WHERE rfc724_mid=?1 OR pre_rfc724_mid=?1 HAVING COUNT(*) > 0 -- Prevent MIN(expr) from returning NULL when there are no rows. ORDER BY timestamp_sent DESC"), (rfc724_mid,), diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index e195a28753..18d1330256 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1518,7 +1518,8 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; CREATE TABLE available_post_msgs ( rfc724_mid TEXT PRIMARY KEY ) STRICT; - ALTER TABLE msgs ADD COLUMN pre_rfc724_mid TEXT DEFAULT '';", + ALTER TABLE msgs ADD COLUMN pre_rfc724_mid TEXT DEFAULT ''; + CREATE INDEX msgs_index9 ON msgs (pre_rfc724_mid);", migration_version, ) .await?; From 78520745939914eb0cbd00e858fd09b2b26e50a5 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Wed, 7 Jan 2026 14:00:11 -0300 Subject: [PATCH 37/39] fix message deletion on IMAP when deleting the whole chat It never worked as expected: if messages showed up on IMAP later, they weren't deleted. Messages must be trashed with the `deleted` flag set, not just deleted from the db. --- deltachat-rpc-client/tests/test_something.py | 41 ++++++++++++++------ src/chat.rs | 10 ++++- src/message.rs | 4 +- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 70e7cd54f6..430933fb07 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -748,29 +748,46 @@ def test_download_small_msg_first(acfactory, tmp_path): assert bob1.wait_for_incoming_msg().get_snapshot().text == "" -def test_delete_available_msg(acfactory, tmp_path, direct_imap): +@pytest.mark.parametrize("delete_chat", [False, True]) +def test_delete_available_msg(acfactory, tmp_path, direct_imap, delete_chat): + """ + Tests `DownloadState.AVAILABLE` message deletion on the receiver side. + Also tests pre- and post-message deletion on the sender side. + """ # Min. UI setting as of v2.35 download_limit = 163840 - alice, bob = acfactory.get_online_accounts(2) - alice.set_config("bcc_self", "1") bob.set_config("download_limit", str(download_limit)) + # Avoid immediate deletion from the server + alice.set_config("bcc_self", "1") + bob.set_config("bcc_self", "1") - chat = alice.create_chat(bob) + chat_alice = alice.create_chat(bob) path = tmp_path / "large" path.write_bytes(os.urandom(download_limit + 1)) - alice_msg = chat.send_file(str(path)) - bob_msg = bob.wait_for_incoming_msg() - assert bob_msg.get_snapshot().download_state == DownloadState.AVAILABLE + msg_alice = chat_alice.send_file(str(path)) + msg_bob = bob.wait_for_incoming_msg() + msg_bob_snapshot = msg_bob.get_snapshot() + assert msg_bob_snapshot.download_state == DownloadState.AVAILABLE + chat_bob = bob.get_chat_by_id(msg_bob_snapshot.chat_id) - bob.delete_messages([bob_msg]) + # Avoid DeleteMessages sync message + bob.set_config("bcc_self", "0") + if delete_chat: + chat_bob.delete() + else: + bob.delete_messages([msg_bob]) alice.wait_for_event(EventType.SMTP_MESSAGE_SENT) alice.wait_for_event(EventType.SMTP_MESSAGE_SENT) - # Avoid DeleteMessages sync message alice.set_config("bcc_self", "0") - alice.delete_messages([alice_msg]) + if delete_chat: + chat_alice.delete() + else: + alice.delete_messages([msg_alice]) for acc in [bob, alice]: - acc.wait_for_event(EventType.MSG_DELETED) + if not delete_chat: + acc.wait_for_event(EventType.MSG_DELETED) + acc_direct_imap = direct_imap(acc) # Messages may be deleted separately while True: acc.wait_for_event(EventType.IMAP_MESSAGE_DELETED) @@ -778,7 +795,7 @@ def test_delete_available_msg(acfactory, tmp_path, direct_imap): event = acc.wait_for_event() if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: break - if len(direct_imap(acc).get_all_messages()) == 0: + if len(acc_direct_imap.get_all_messages()) == 0: break diff --git a/src/chat.rs b/src/chat.rs index c328292e2b..0b6444a16d 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -643,7 +643,15 @@ impl ChatId { "DELETE FROM msgs_mdns WHERE msg_id IN (SELECT id FROM msgs WHERE chat_id=?)", (self,), )?; - transaction.execute("DELETE FROM msgs WHERE chat_id=?", (self,))?; + // If you change which information is preserved here, also change `MsgId::trash()` + // and other places it references. + transaction.execute( + " +INSERT OR REPLACE INTO msgs (id, rfc724_mid, pre_rfc724_mid, timestamp, chat_id, deleted) +SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=? + ", + (DC_CHAT_ID_TRASH, self), + )?; transaction.execute("DELETE FROM chats_contacts WHERE chat_id=?", (self,))?; transaction.execute("DELETE FROM chats WHERE id=?", (self,))?; Ok(()) diff --git a/src/message.rs b/src/message.rs index d8c971cc17..1cf04e540e 100644 --- a/src/message.rs +++ b/src/message.rs @@ -131,8 +131,8 @@ impl MsgId { .sql .execute( // If you change which information is preserved here, also change - // `delete_expired_messages()` and which information `receive_imf::add_parts()` - // still adds to the db if chat_id is TRASH. + // `ChatId::delete_ex()`, `delete_expired_messages()` and which information + // `receive_imf::add_parts()` still adds to the db if chat_id is TRASH. " INSERT OR REPLACE INTO msgs (id, rfc724_mid, pre_rfc724_mid, timestamp, chat_id, deleted) SELECT ?1, rfc724_mid, pre_rfc724_mid, timestamp, ?, ? FROM msgs WHERE id=?1 From 1f4cb7272564581dad975fdf650f1fbed22a3c11 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Thu, 8 Jan 2026 12:11:28 -0300 Subject: [PATCH 38/39] test_delete_fully_downloaded_msg --- deltachat-rpc-client/tests/test_something.py | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 430933fb07..f9d7171cb6 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -799,6 +799,42 @@ def test_delete_available_msg(acfactory, tmp_path, direct_imap, delete_chat): break +def test_delete_fully_downloaded_msg(acfactory, tmp_path, direct_imap): + alice, bob = acfactory.get_online_accounts(2) + # Avoid immediate deletion from the server + bob.set_config("bcc_self", "1") + + chat_alice = alice.create_chat(bob) + path = tmp_path / "large" + # Big enough to be sent with a pre-message + path.write_bytes(os.urandom(300000)) + chat_alice.send_file(str(path)) + + msg = bob.wait_for_incoming_msg() + msg_snapshot = msg.get_snapshot() + assert msg_snapshot.download_state == DownloadState.AVAILABLE + msgs_changed_event = bob.wait_for_msgs_changed_event() + assert msgs_changed_event.msg_id == msg.id + msg_snapshot = msg.get_snapshot() + assert msg_snapshot.download_state == DownloadState.DONE + + bob_direct_imap = direct_imap(bob) + assert len(bob_direct_imap.get_all_messages()) == 2 + # Avoid DeleteMessages sync message + bob.set_config("bcc_self", "0") + bob.delete_messages([msg]) + bob.wait_for_event(EventType.MSG_DELETED) + # Messages may be deleted separately + while True: + bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) + while True: + event = bob.wait_for_event() + if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: + break + if len(bob_direct_imap.get_all_messages()) == 0: + break + + def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages From 5d62eb8abfcce9d72f4321dd66c6e8e0c9fcd420 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Thu, 8 Jan 2026 12:12:20 -0300 Subject: [PATCH 39/39] fix: Delete received message on IMAP immediately if this is configured I.e. if DeleteServerAfter is set or BccSelf is unset (single-device mode) for chatmail. Looks like this never worked as expected. --- deltachat-rpc-client/tests/test_something.py | 29 ++++++++++++++++++++ src/receive_imf.rs | 1 + 2 files changed, 30 insertions(+) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index f9d7171cb6..476cc57acd 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -835,6 +835,35 @@ def test_delete_fully_downloaded_msg(acfactory, tmp_path, direct_imap): break +def test_imap_autodelete_fully_downloaded_msg(acfactory, tmp_path, direct_imap): + alice, bob = acfactory.get_online_accounts(2) + + chat_alice = alice.create_chat(bob) + path = tmp_path / "large" + # Big enough to be sent with a pre-message + path.write_bytes(os.urandom(300000)) + chat_alice.send_file(str(path)) + + msg = bob.wait_for_incoming_msg() + msg_snapshot = msg.get_snapshot() + assert msg_snapshot.download_state == DownloadState.AVAILABLE + msgs_changed_event = bob.wait_for_msgs_changed_event() + assert msgs_changed_event.msg_id == msg.id + msg_snapshot = msg.get_snapshot() + assert msg_snapshot.download_state == DownloadState.DONE + + bob_direct_imap = direct_imap(bob) + # Messages may be deleted separately + while True: + if len(bob_direct_imap.get_all_messages()) == 0: + break + bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED) + while True: + event = bob.wait_for_event() + if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg: + break + + def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages diff --git a/src/receive_imf.rs b/src/receive_imf.rs index effa98aed3..7f1d2aaf0f 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -950,6 +950,7 @@ pub(crate) async fn receive_imf_inner( ), ) .await?; + context.scheduler.interrupt_inbox().await; } if target.is_none() && !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {