diff --git a/src/mimefactory.rs b/src/mimefactory.rs
index 1c7b564d3d..afec20144b 100644
--- a/src/mimefactory.rs
+++ b/src/mimefactory.rs
@@ -426,8 +426,16 @@ impl MimeFactory {
                     },
                 )
                 .await?;
-            let recipient_ids: Vec<_> = recipient_ids.into_iter().collect();
-            ContactId::scaleup_origin(context, &recipient_ids, Origin::OutgoingTo).await?;
+            let recipient_ids: Vec<_> = recipient_ids
+                .into_iter()
+                .filter(|id| *id != ContactId::SELF)
+                .collect();
+            if recipient_ids.len() == 1
+                && msg.param.get_cmd() != SystemMessage::MemberRemovedFromGroup
+                && chat.typ != Chattype::OutBroadcast
+            {
+                ContactId::scaleup_origin(context, &recipient_ids, Origin::OutgoingTo).await?;
+            }
 
             if !msg.is_system_message()
                 && msg.param.get_int(Param::Reaction).unwrap_or_default() == 0
diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs
index 151637f8e8..b29e64b4a6 100644
--- a/src/receive_imf/receive_imf_tests.rs
+++ b/src/receive_imf/receive_imf_tests.rs
@@ -3878,6 +3878,29 @@ async fn test_group_contacts_goto_bottom() -> Result<()> {
     let contacts = Contact::get_all(bob, 0, None).await?;
     assert_eq!(contacts.len(), 2);
     assert_eq!(contacts[0], bob_fiona_id);
+
+    send_text_msg(
+        bob,
+        bob_chat_id,
+        "Hi Alice, stay down in my contact list".to_string(),
+    )
+    .await?;
+    bob.pop_sent_msg().await;
+    let contacts = Contact::get_all(bob, 0, None).await?;
+    assert_eq!(contacts[0], bob_fiona_id);
+
+    remove_contact_from_chat(bob, bob_chat_id, bob_fiona_id).await?;
+    bob.pop_sent_msg().await;
+    let contacts = Contact::get_all(bob, 0, None).await?;
+    // Fiona is still the 0th contact. This makes sense, maybe Bob is going to remove Alice from the
+    // chat too, so no need to make Alice a more "important" contact yet.
+    assert_eq!(contacts[0], bob_fiona_id);
+
+    send_text_msg(bob, bob_chat_id, "Alice, jump up!".to_string()).await?;
+    bob.pop_sent_msg().await;
+    let contacts = Contact::get_all(bob, 0, None).await?;
+    let bob_alice_id = bob.add_or_lookup_contact_id(alice).await;
+    assert_eq!(contacts[0], bob_alice_id);
     Ok(())
 }