diff --git a/chromadb/segment/impl/manager/local.py b/chromadb/segment/impl/manager/local.py
index 907db7f3887..83ab16cfaa4 100644
--- a/chromadb/segment/impl/manager/local.py
+++ b/chromadb/segment/impl/manager/local.py
@@ -157,9 +157,9 @@ def delete_segments(self, collection_id: UUID) -> Sequence[UUID]:
         segments = self._sysdb.get_segments(collection=collection_id)
         for segment in segments:
             collection_id = segment["collection"]
-            self._vector_instances_file_handle_cache.evict(collection_id)
             if segment["id"] in self._instances:
                 if segment["type"] == SegmentType.HNSW_LOCAL_PERSISTED.value:
+                    self._vector_instances_file_handle_cache.evict(collection_id)
                     instance = self.get_segment(collection_id, VectorReader)
                     instance.delete()
                 elif segment["type"] == SegmentType.SQLITE.value: