diff --git a/src/collision/collider/backend.rs b/src/collision/collider/backend.rs
index c2f002b98..9e0b3f995 100644
--- a/src/collision/collider/backend.rs
+++ b/src/collision/collider/backend.rs
@@ -183,7 +183,7 @@ impl<C: ScalableCollider> Plugin for ColliderBackendPlugin<C> {
                 world
                     .commands()
                     .entity(ctx.entity)
-                    .remove::<ColliderMarker>();
+                    .try_remove::<ColliderMarker>();
 
                 let entity_ref = world.entity_mut(ctx.entity);
 
diff --git a/src/collision/collider/collider_hierarchy/mod.rs b/src/collision/collider/collider_hierarchy/mod.rs
index 15caf958e..34e325992 100644
--- a/src/collision/collider/collider_hierarchy/mod.rs
+++ b/src/collision/collider/collider_hierarchy/mod.rs
@@ -9,7 +9,7 @@ use crate::prelude::*;
 use bevy::{
     ecs::{
         component::HookContext,
-        relationship::{Relationship, RelationshipHookMode},
+        relationship::{Relationship, RelationshipHookMode, RelationshipSourceCollection},
         world::DeferredWorld,
     },
     prelude::*,
@@ -127,6 +127,57 @@ impl Relationship for ColliderOf {
             );
         }
     }
+
+    fn on_replace(
+        mut world: DeferredWorld,
+        HookContext {
+            entity,
+            relationship_hook_mode,
+            ..
+        }: HookContext,
+    ) {
+        // This is largely the same as the default implementation,
+        // but does not panic if the relationship target does not exist.
+
+        match relationship_hook_mode {
+            RelationshipHookMode::Run => {}
+            RelationshipHookMode::Skip => return,
+            RelationshipHookMode::RunIfNotLinked => {
+                if <Self::RelationshipTarget as RelationshipTarget>::LINKED_SPAWN {
+                    return;
+                }
+            }
+        }
+        let rigid_body = world.entity(entity).get::<Self>().unwrap().get();
+        if let Ok(mut rigid_body_mut) = world.get_entity_mut(rigid_body) {
+            if let Some(mut relationship_target) =
+                rigid_body_mut.get_mut::<Self::RelationshipTarget>()
+            {
+                RelationshipSourceCollection::remove(
+                    relationship_target.collection_mut_risky(),
+                    entity,
+                );
+                if relationship_target.len() == 0 {
+                    if let Ok(mut entity) = world.commands().get_entity(rigid_body) {
+                        // this "remove" operation must check emptiness because in the event that an identical
+                        // relationship is inserted on top, this despawn would result in the removal of that identical
+                        // relationship ... not what we want!
+                        entity.queue_handled(
+                            |mut entity: EntityWorldMut| {
+                                if entity
+                                    .get::<Self::RelationshipTarget>()
+                                    .is_some_and(RelationshipTarget::is_empty)
+                                {
+                                    entity.remove::<Self::RelationshipTarget>();
+                                }
+                            },
+                            |_, _| {},
+                        );
+                    }
+                }
+            }
+        }
+    }
 }
 
 /// A [`RelationshipTarget`] component that tracks which colliders are attached to a [`RigidBody`].
diff --git a/src/collision/collider/collider_hierarchy/plugin.rs b/src/collision/collider/collider_hierarchy/plugin.rs
index bd627ff66..8d7bcc198 100644
--- a/src/collision/collider/collider_hierarchy/plugin.rs
+++ b/src/collision/collider/collider_hierarchy/plugin.rs
@@ -36,7 +36,7 @@ impl Plugin for ColliderHierarchyPlugin {
 
                 // Make sure the collider is on the same entity as the rigid body.
                 if query.contains(entity) {
-                    commands.entity(entity).remove::<ColliderOf>();
+                    commands.entity(entity).try_remove::<ColliderOf>();
                 }
             },
         );
@@ -117,7 +117,7 @@ fn on_rigid_body_removed(
         for collider_entity in colliders.iter() {
             commands
                 .entity(collider_entity)
-                .remove::<(ColliderOf, ColliderTransform)>();
+                .try_remove::<(ColliderOf, ColliderTransform)>();
         }
     }
 }