Add useKrispNoiseCancellation hook (#986)

.changeset/wise-doors-jump.md

@@ -0,0 +1,6 @@
+---
+"@livekit/components-core": patch
+"@livekit/components-react": patch
+---
+
+Add krisp hook

packages/core/src/components/trackMutedIndicator.ts

@@ -1,3 +1,4 @@
+// @ts-ignore some module resolutions (other than 'node') choke on this
 import type { Styles } from '@livekit/components-styles/dist/types_unprefixed/index.scss';
 import { Track } from 'livekit-client';
 import { mutedObserver } from '../observables/participant';

packages/core/src/observables/participant.ts

@@ -1,6 +1,7 @@
 import type { ParticipantPermission } from '@livekit/protocol';
 import { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client';
 import { ParticipantEvent, RoomEvent, Track } from 'livekit-client';
+// @ts-ignore some module resolutions (other than 'node') choke on this
 import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant';
 import type { Subscriber } from 'rxjs';
 import { Observable, map, startWith, switchMap } from 'rxjs';

packages/core/src/observables/room.ts

@@ -2,6 +2,7 @@ import type { Subscriber, Subscription } from 'rxjs';
 import { Subject, map, Observable, startWith, finalize, filter, concat } from 'rxjs';
 import type { Participant, TrackPublication } from 'livekit-client';
 import { LocalParticipant, Room, RoomEvent, Track } from 'livekit-client';
+// @ts-ignore some module resolutions (other than 'node') choke on this
 import type { RoomEventCallbacks } from 'livekit-client/dist/src/room/Room';
 import { log } from '../logger';
 export function observeRoomEvents(room: Room, ...events: RoomEvent[]): Observable<Room> {

packages/core/src/observables/track.ts

@@ -14,6 +14,7 @@ import type { TrackReference } from '../track-reference';
 import { observeRoomEvents } from './room';
 import type { ParticipantTrackIdentifier } from '../types';
 import { observeParticipantEvents } from './participant';
+// @ts-ignore some module resolutions (other than 'node') choke on this
 import type { PublicationEventCallbacks } from 'livekit-client/dist/src/room/track/TrackPublication';
 
 export function trackObservable(track: TrackPublication) {

packages/react/.size-limit.js

@@ -4,20 +4,20 @@ module.exports = [
     path: 'dist/index.mjs',
     import: '{ LiveKitRoom }',
     limit: '4 kB',
-    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'],
+    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'],
   },
   {
     name: 'LiveKitRoom with VideoConference',
     path: 'dist/index.mjs',
     import: '{ LiveKitRoom, VideoConference }',
     limit: '40 kB',
-    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'],
+    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'],
   },
   {
     name: 'All exports',
     path: 'dist/index.mjs',
     import: '*',
     limit: '100 kB',
-    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'],
+    ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'],
   },
 ];

packages/react/etc/components-react.api.md

@@ -12,12 +12,14 @@ import { ConnectionState as ConnectionState_2 } from 'livekit-client';
 import type { CreateLocalTracksOptions } from 'livekit-client';
 import type { DataPublishOptions } from 'livekit-client';
 import type { HTMLAttributes } from 'react';
+import type { KrispNoiseFilterProcessor } from '@livekit/krisp-noise-filter';
 import { LocalAudioTrack } from 'livekit-client';
 import { LocalParticipant } from 'livekit-client';
 import type { LocalTrack } from 'livekit-client';
 import { LocalTrackPublication } from 'livekit-client';
 import { LocalVideoTrack } from 'livekit-client';
 import type { MediaDeviceFailure } from 'livekit-client';
+import type { NoiseFilterOptions } from '@livekit/krisp-noise-filter';
 import { Participant } from 'livekit-client';
 import type { ParticipantEvent } from 'livekit-client';
 import type { ParticipantKind } from 'livekit-client';
@@ -883,6 +885,21 @@ export function useIsRecording(room?: Room): boolean;
 // @public
 export function useIsSpeaking(participant?: Participant): boolean;
 
+// @alpha
+export function useKrispNoiseFilter(options?: useKrispNoiseFilterOptions): {
+    setNoiseFilterEnabled: (enable: boolean) => Promise<void>;
+    isNoiseFilterEnabled: boolean;
+    isNoiseFilterPending: boolean;
+    processor: KrispNoiseFilterProcessor | undefined;
+};
+
+// @alpha (undocumented)
+export interface useKrispNoiseFilterOptions {
+    // (undocumented)
+    filterOptions?: NoiseFilterOptions;
+    trackRef?: TrackReferenceOrPlaceholder;
+}
+
 // @public
 export function useLayoutContext(): LayoutContextType;
 

packages/react/package.json

@@ -52,11 +52,17 @@
     "usehooks-ts": "3.1.0"
   },
   "peerDependencies": {
+    "@livekit/krisp-noise-filter": "^0.2.12",
     "livekit-client": "^2.5.4",
     "react": ">=18",
     "react-dom": ">=18",
     "tslib": "^2.6.2"
   },
+  "peerDependenciesMeta": {
+    "@livekit/krisp-noise-filter": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@livekit/protocol": "^1.22.0",
     "@microsoft/api-extractor": "^7.35.0",

packages/react/src/hooks/cloud/krisp/useKrispNoiseFilter.ts

@@ -0,0 +1,104 @@
+import * as React from 'react';
+import { LocalAudioTrack } from 'livekit-client';
+import type { KrispNoiseFilterProcessor, NoiseFilterOptions } from '@livekit/krisp-noise-filter';
+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core';
+import { useLocalParticipant } from '../../useLocalParticipant';
+
+/**
+ * @alpha
+ */
+export interface useKrispNoiseFilterOptions {
+  /**
+   * by default the hook will use the localParticipant's microphone track publication.
+   * You can override this behavior by passing in a target TrackReference here
+   */
+  trackRef?: TrackReferenceOrPlaceholder;
+  filterOptions?: NoiseFilterOptions;
+}
+
+/**
+ * This hook is a convenience helper for enabling Krisp Enhanced Audio Noise Cancellation on LiveKit audio tracks.
+ * It returns a `setNoiseFilterEnabled` method to conveniently toggle between enabled and disabled states.
+ *
+ * @remarks Krisp noise filter is a feature that's only supported on LiveKit cloud plans
+ * @alpha
+ * @example
+ * ```tsx
+ *   const krisp = useKrispNoiseFilter();
+ *   return <input
+        type="checkbox"
+        onChange={(ev) => krisp.setNoiseFilterEnabled(ev.target.checked)}
+        checked={krisp.isNoiseFilterEnabled}
+        disabled={krisp.isNoiseFilterPending}
+      />
+ * ```
+ */
+export function useKrispNoiseFilter(options: useKrispNoiseFilterOptions = {}) {
+  const [shouldEnable, setShouldEnable] = React.useState(false);
+  const [isNoiseFilterPending, setIsNoiseFilterPending] = React.useState(false);
+  const [isNoiseFilterEnabled, setIsNoiseFilterEnabled] = React.useState(false);
+  let micPublication = useLocalParticipant().microphoneTrack;
+  const [krispProcessor, setKrispProcessor] = React.useState<
+    KrispNoiseFilterProcessor | undefined
+  >();
+  if (options.trackRef) {
+    micPublication = options.trackRef.publication;
+  }
+
+  const setNoiseFilterEnabled = React.useCallback(async (enable: boolean) => {
+    if (enable) {
+      const { KrispNoiseFilter, isKrispNoiseFilterSupported } = await import(
+        '@livekit/krisp-noise-filter'
+      );
+
+      if (!isKrispNoiseFilterSupported()) {
+        console.warn('Krisp noise filter is not supported in this browser');
+        return;
+      }
+      if (!krispProcessor) {
+        setKrispProcessor(KrispNoiseFilter(options.filterOptions));
+      }
+    }
+    setShouldEnable((prev) => {
+      if (prev !== enable) {
+        setIsNoiseFilterPending(true);
+      }
+      return enable;
+    });
+  }, []);
+
+  React.useEffect(() => {
+    if (micPublication && micPublication.track instanceof LocalAudioTrack && krispProcessor) {
+      const currentProcessor = micPublication.track.getProcessor();
+      if (currentProcessor && currentProcessor.name === 'livekit-noise-filter') {
+        setIsNoiseFilterPending(true);
+        (currentProcessor as KrispNoiseFilterProcessor).setEnabled(shouldEnable).finally(() => {
+          setIsNoiseFilterPending(false);
+          setIsNoiseFilterEnabled(shouldEnable);
+        });
+      } else if (!currentProcessor && shouldEnable) {
+        setIsNoiseFilterPending(true);
+        micPublication?.track
+          ?.setProcessor(krispProcessor)
+          .then(() => krispProcessor.setEnabled(shouldEnable))
+          .then(() => {
+            setIsNoiseFilterEnabled(true);
+          })
+          .catch((e: any) => {
+            setIsNoiseFilterEnabled(false);
+            console.error(e);
+          })
+          .finally(() => {
+            setIsNoiseFilterPending(false);
+          });
+      }
+    }
+  }, [shouldEnable, micPublication, krispProcessor]);
+
+  return {
+    setNoiseFilterEnabled,
+    isNoiseFilterEnabled,
+    isNoiseFilterPending,
+    processor: krispProcessor,
+  };
+}

packages/react/src/hooks/index.ts

@@ -54,3 +54,4 @@ export * from './useTrackTranscription';
 export * from './useVoiceAssistant';
 export * from './useParticipantAttributes';
 export * from './useIsRecording';
+export { useKrispNoiseFilter, useKrispNoiseFilterOptions } from './cloud/krisp/useKrispNoiseFilter';

packages/react/src/hooks/useParticipantAttributes.ts

@@ -57,7 +57,7 @@ export function useParticipantAttribute(
     }
     const subscription = participantAttributesObserver(p).subscribe((val) => {
       if (val.changed[attributeKey] !== undefined) {
-        setAttribute(val.changed[attributeKey]);
+        setAttribute(val.attributes[attributeKey]);
       }
     });
     return () => {

packages/react/tsup.config.ts

@@ -5,5 +5,5 @@ import defaults from '../../tsup.config';
 export default defineConfig({
   ...defaults,
   entry: ['src/index.ts', 'src/hooks/index.ts', 'src/prefabs/index.ts'],
-  external: ['livekit-client', 'react', 'react-dom'],
+  external: ['livekit-client', 'react', 'react-dom', '@livekit/krisp-noise-filter'],
 });