diff --git a/src/set.rs b/src/set.rs
index 2c9e7f22c..a1254cba6 100644
--- a/src/set.rs
+++ b/src/set.rs
@@ -1659,11 +1659,15 @@ where
     /// ```
     fn bitxor_assign(&mut self, rhs: &HashSet<T, S>) {
         for item in rhs {
-            if self.contains(item) {
-                self.remove(item);
-            } else {
-                self.insert(item.clone());
-            }
+            let entry = self.map.raw_entry_mut().from_key(item);
+            match entry {
+                map::RawEntryMut::Occupied(e) => {
+                    e.remove();
+                }
+                map::RawEntryMut::Vacant(e) => {
+                    e.insert(item.to_owned(), ());
+                }
+            };
         }
     }
 }