diff --git a/web/src/lib/components/faces-page/person-side-panel.svelte b/web/src/lib/components/faces-page/person-side-panel.svelte
index aded8ae0f..b2e2cf4c7 100644
--- a/web/src/lib/components/faces-page/person-side-panel.svelte
+++ b/web/src/lib/components/faces-page/person-side-panel.svelte
@@ -151,8 +151,10 @@
 
   const handleAddRemovedFace = (indexToRemove: number) => {
     $boundingBoxesArray = [];
+    unassignedFaces = unassignedFaces.map((obj) =>
+      obj && obj.id === selectedPersonToUnassign[indexToRemove].id ? null : obj,
+    );
     selectedPersonToUnassign = selectedPersonToUnassign.filter((_, index) => index !== indexToRemove);
-    peopleWithFaces = peopleWithFaces
   };
 
   const handleUnassignFaces = async () => {