diff --git a/src/receive_imf.rs b/src/receive_imf.rs
index 181cf07f97..c332ec004b 100644
--- a/src/receive_imf.rs
+++ b/src/receive_imf.rs
@@ -2182,6 +2182,8 @@ async fn apply_group_changes(
 
     let mut chat_contacts =
         HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat_id).await?);
+    let is_from_in_chat =
+        !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
 
     // Whether to rebuild the member list from scratch.
     let recreate_member_list = self_added;
@@ -2317,7 +2319,7 @@ async fn apply_group_changes(
                 expected_timestamps_count
             );
         }
-    } else {
+    } else if is_from_in_chat {
         let mut new_members = HashSet::from_iter(to_ids.iter().copied());
         new_members.insert(ContactId::SELF);
         if !from_id.is_special() {