diff --git a/packages/reactivity/__tests__/reactive.spec.ts b/packages/reactivity/__tests__/reactive.spec.ts
index aabd954568a..a7e71cfbd2b 100644
--- a/packages/reactivity/__tests__/reactive.spec.ts
+++ b/packages/reactivity/__tests__/reactive.spec.ts
@@ -112,6 +112,19 @@ describe('reactivity/reactive', () => {
     expect(dummy).toBe(false)
   })
 
+  // #8647
+  test('observing Set with reactive initial value', () => {
+    const observed = reactive({})
+    const observedSet = reactive(new Set([observed]))
+
+    expect(observedSet.has(observed)).toBe(true)
+    expect(observedSet.size).toBe(1)
+
+    // expect nothing happens
+    observedSet.add(observed)
+    expect(observedSet.size).toBe(1)
+  })
+
   test('observed value should proxy mutations to original (Object)', () => {
     const original: any = { foo: 1 }
     const observed = reactive(original)
diff --git a/packages/reactivity/src/collectionHandlers.ts b/packages/reactivity/src/collectionHandlers.ts
index 048b7f38863..e110b976832 100644
--- a/packages/reactivity/src/collectionHandlers.ts
+++ b/packages/reactivity/src/collectionHandlers.ts
@@ -167,15 +167,18 @@ function createInstrumentations(
         }
       : {
           add(this: SetTypes, value: unknown) {
-            if (!shallow && !isShallow(value) && !isReadonly(value)) {
-              value = toRaw(value)
-            }
             const target = toRaw(this)
             const proto = getProto(target)
-            const hadKey = proto.has.call(target, value)
+            const valueToAdd =
+              !shallow && !isShallow(value) && !isReadonly(value)
+                ? toRaw(value)
+                : value
+            const hadKey =
+              proto.has.call(target, valueToAdd) ||
+              (value !== valueToAdd && proto.has.call(target, value))
             if (!hadKey) {
-              target.add(value)
-              trigger(target, TriggerOpTypes.ADD, value, value)
+              target.add(valueToAdd)
+              trigger(target, TriggerOpTypes.ADD, valueToAdd, valueToAdd)
             }
             return this
           },