From 495e2ef0d8077d1f74cdaa7a4139b503e9d9b85f Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 29 Sep 2023 19:06:34 +0200 Subject: [PATCH 01/11] feat: ScreenShare Audio support --- packages/client/src/Call.ts | 51 ++- packages/client/src/devices/devices.ts | 11 +- .../client/src/helpers/DynascaleManager.ts | 13 +- .../__tests__/DynascaleManager.test.ts | 1 + .../src/helpers/__tests__/hq-audio-sdp.ts | 332 ++++++++++++++++++ .../src/helpers/__tests__/sdp-munging.test.ts | 15 +- packages/client/src/helpers/sdp-munging.ts | 49 +++ packages/client/src/rtc/Publisher.ts | 111 +++--- packages/client/src/rtc/Subscriber.ts | 6 + packages/client/src/rtc/helpers/tracks.ts | 22 +- packages/client/src/types.ts | 12 + .../src/core/components/Audio/Audio.tsx | 27 +- .../components/Audio/ParticipantsAudio.tsx | 32 +- .../ParticipantView/ParticipantView.tsx | 17 +- 14 files changed, 622 insertions(+), 77 deletions(-) create mode 100644 packages/client/src/helpers/__tests__/hq-audio-sdp.ts diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 43024deaae..4bf57aace0 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -68,6 +68,7 @@ import { } from './gen/coordinator'; import { join, reconcileParticipantLocalState } from './rtc/flows/join'; import { + AudioTrackType, CallConstructor, CallLeaveOptions, DebounceType, @@ -76,6 +77,7 @@ import { StreamVideoParticipant, StreamVideoParticipantPatches, SubscriptionChanges, + TrackMuteType, VideoTrackType, VisibilityState, } from './types'; @@ -768,9 +770,21 @@ export class Call { const { audioStream, videoStream, - screenShareStream: screenShare, + screenShareStream, + screenShareAudioStream, } = localParticipant; + let screenShare: MediaStream | undefined; + if (screenShareStream || screenShareAudioStream) { + screenShare = new MediaStream(); + screenShareStream?.getVideoTracks().forEach((track) => { + screenShare?.addTrack(track); + }); + screenShareAudioStream?.getAudioTracks().forEach((track) => { + screenShare?.addTrack(track); + }); + } + // restore previous publishing state if (audioStream) await this.publishAudioStream(audioStream); if (videoStream) await this.publishVideoStream(videoStream); @@ -1141,6 +1155,15 @@ export class Call { screenShareTrack, TrackType.SCREEN_SHARE, ); + + const [screenShareAudioTrack] = screenShareStream.getAudioTracks(); + if (screenShareAudioTrack) { + await this.publisher.publishStream( + screenShareStream, + screenShareAudioTrack, + TrackType.SCREEN_SHARE_AUDIO, + ); + } }; /** @@ -1252,6 +1275,13 @@ export class Call { dimension: p.screenShareDimension, }); } + if (p.publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO)) { + subscriptions.push({ + userId: p.userId, + sessionId: p.sessionId, + trackType: TrackType.SCREEN_SHARE_AUDIO, + }); + } } // schedule update this.trackSubscriptionsSubject.next({ type, data: subscriptions }); @@ -1414,7 +1444,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteSelf = (type: 'audio' | 'video' | 'screenshare') => { + muteSelf = (type: TrackMuteType) => { const myUserId = this.currentUserId; if (myUserId) { return this.muteUser(myUserId, type); @@ -1426,7 +1456,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteOthers = (type: 'audio' | 'video' | 'screenshare') => { + muteOthers = (type: TrackMuteType) => { const trackType = muteTypeToTrackType(type); if (!trackType) return; const userIdsToMute: string[] = []; @@ -1445,10 +1475,7 @@ export class Call { * @param userId the id of the user to mute. * @param type the type of the mute operation. */ - muteUser = ( - userId: string | string[], - type: 'audio' | 'video' | 'screenshare', - ) => { + muteUser = (userId: string | string[], type: TrackMuteType) => { return this.streamClient.post( `${this.streamClientBasePath}/mute_users`, { @@ -1463,7 +1490,7 @@ export class Call { * * @param type the type of the mute operation. */ - muteAllUsers = (type: 'audio' | 'video' | 'screenshare') => { + muteAllUsers = (type: TrackMuteType) => { return this.streamClient.post( `${this.streamClientBasePath}/mute_users`, { @@ -1952,11 +1979,17 @@ export class Call { * * @param audioElement the audio element to bind to. * @param sessionId the session id. + * @param trackType the kind of audio. */ - bindAudioElement = (audioElement: HTMLAudioElement, sessionId: string) => { + bindAudioElement = ( + audioElement: HTMLAudioElement, + sessionId: string, + trackType: AudioTrackType = 'audioTrack', + ) => { const unbind = this.dynascaleManager.bindAudioElement( audioElement, sessionId, + trackType, ); if (!unbind) return; diff --git a/packages/client/src/devices/devices.ts b/packages/client/src/devices/devices.ts index 73bc5b101f..6e0c3bdab5 100644 --- a/packages/client/src/devices/devices.ts +++ b/packages/client/src/devices/devices.ts @@ -199,7 +199,16 @@ export const getScreenShareStream = async ( try { return await navigator.mediaDevices.getDisplayMedia({ video: true, - audio: false, + audio: { + channelCount: { + ideal: 2, + }, + echoCancellation: false, + autoGainControl: false, + noiseSuppression: false, + }, + // @ts-expect-error - not present in types yet + systemAudio: 'include', ...options, }); } catch (e) { diff --git a/packages/client/src/helpers/DynascaleManager.ts b/packages/client/src/helpers/DynascaleManager.ts index e8f57593e7..620be0c13b 100644 --- a/packages/client/src/helpers/DynascaleManager.ts +++ b/packages/client/src/helpers/DynascaleManager.ts @@ -1,5 +1,6 @@ import { Call } from '../Call'; import { + AudioTrackType, DebounceType, StreamVideoLocalParticipant, StreamVideoParticipant, @@ -324,9 +325,14 @@ export class DynascaleManager { * * @param audioElement the audio element to bind to. * @param sessionId the session id. + * @param trackType the kind of audio. * @returns a cleanup function that will unbind the audio element. */ - bindAudioElement = (audioElement: HTMLAudioElement, sessionId: string) => { + bindAudioElement = ( + audioElement: HTMLAudioElement, + sessionId: string, + trackType: AudioTrackType, + ) => { const participant = this.call.state.findParticipantBySessionId(sessionId); if (!participant || participant.isLocalParticipant) return; @@ -345,7 +351,10 @@ export class DynascaleManager { const updateMediaStreamSubscription = participant$ .pipe(distinctUntilKeyChanged('audioStream')) .subscribe((p) => { - const source = p.audioStream; + const source = + trackType === 'screenShareAudioTrack' + ? p.screenShareAudioStream + : p.audioStream; if (audioElement.srcObject === source) return; setTimeout(() => { diff --git a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts index 3b8753afc1..f73f7921d5 100644 --- a/packages/client/src/helpers/__tests__/DynascaleManager.test.ts +++ b/packages/client/src/helpers/__tests__/DynascaleManager.test.ts @@ -128,6 +128,7 @@ describe('DynascaleManager', () => { const cleanup = dynascaleManager.bindAudioElement( audioElement, 'session-id', + 'audioTrack', ); expect(audioElement.autoplay).toBe(true); diff --git a/packages/client/src/helpers/__tests__/hq-audio-sdp.ts b/packages/client/src/helpers/__tests__/hq-audio-sdp.ts new file mode 100644 index 0000000000..c4bad821df --- /dev/null +++ b/packages/client/src/helpers/__tests__/hq-audio-sdp.ts @@ -0,0 +1,332 @@ +export const initialSdp = ` +v=0 +o=- 898697271686242868 5 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 1 2 3 +a=extmap-allow-mixed +a=msid-semantic: WMS e893e3ad-d9e8-4b56-998f-0d89213dd857 +m=video 60017 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 +c=IN IP4 79.125.240.146 +a=rtcp:9 IN IP4 0.0.0.0 +a=candidate:2824354870 1 udp 2122260223 192.168.1.102 60017 typ host generation 0 network-id 2 +a=candidate:427386432 1 udp 2122194687 192.168.1.244 63221 typ host generation 0 network-id 1 network-cost 10 +a=candidate:2841136656 1 udp 1686052607 79.125.240.146 60017 typ srflx raddr 192.168.1.102 rport 60017 generation 0 network-id 2 +a=candidate:410588262 1 udp 1685987071 79.125.240.146 63221 typ srflx raddr 192.168.1.244 rport 63221 generation 0 network-id 1 network-cost 10 +a=candidate:3600277166 1 tcp 1518280447 192.168.1.102 9 typ host tcptype active generation 0 network-id 2 +a=candidate:1740014808 1 tcp 1518214911 192.168.1.244 9 typ host tcptype active generation 0 network-id 1 network-cost 10 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:e893e3ad-d9e8-4b56-998f-0d89213dd857 44e5d53f-ce6a-4bf4-824a-335113d86e6e +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 H264/90000 +a=rtcp-fb:112 goog-remb +a=rtcp-fb:112 transport-cc +a=rtcp-fb:112 ccm fir +a=rtcp-fb:112 nack +a=rtcp-fb:112 nack pli +a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:116 red/90000 +a=rtpmap:117 rtx/90000 +a=fmtp:117 apt=116 +a=rtpmap:118 ulpfec/90000 +a=rid:q send +a=rid:h send +a=rid:f send +a=simulcast:send q;h;f +m=audio 9 UDP/TLS/RTP/SAVPF 63 111 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:1 +a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f +a=rtcp-mux +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:743121750 cname:mbGW+aeWMLFhPbBC +a=ssrc:743121750 msid:- cb02416c-8035-4e80-8b7e-becd6b7f600f +m=video 9 UDP/TLS/RTP/SAVPF 96 97 102 103 104 105 106 107 108 109 127 125 39 40 45 46 98 99 100 101 112 113 116 117 118 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:2 +a=extmap:1 urn:ietf:params:rtp-hdrext:toffset +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:3 urn:3gpp:video-orientation +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay +a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type +a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing +a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +a=rtcp-mux +a=rtcp-rsize +a=rtpmap:96 VP8/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=rtpmap:97 rtx/90000 +a=fmtp:97 apt=96 +a=rtpmap:102 H264/90000 +a=rtcp-fb:102 goog-remb +a=rtcp-fb:102 transport-cc +a=rtcp-fb:102 ccm fir +a=rtcp-fb:102 nack +a=rtcp-fb:102 nack pli +a=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f +a=rtpmap:103 rtx/90000 +a=fmtp:103 apt=102 +a=rtpmap:104 H264/90000 +a=rtcp-fb:104 goog-remb +a=rtcp-fb:104 transport-cc +a=rtcp-fb:104 ccm fir +a=rtcp-fb:104 nack +a=rtcp-fb:104 nack pli +a=fmtp:104 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f +a=rtpmap:105 rtx/90000 +a=fmtp:105 apt=104 +a=rtpmap:106 H264/90000 +a=rtcp-fb:106 goog-remb +a=rtcp-fb:106 transport-cc +a=rtcp-fb:106 ccm fir +a=rtcp-fb:106 nack +a=rtcp-fb:106 nack pli +a=fmtp:106 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +a=rtpmap:107 rtx/90000 +a=fmtp:107 apt=106 +a=rtpmap:108 H264/90000 +a=rtcp-fb:108 goog-remb +a=rtcp-fb:108 transport-cc +a=rtcp-fb:108 ccm fir +a=rtcp-fb:108 nack +a=rtcp-fb:108 nack pli +a=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f +a=rtpmap:109 rtx/90000 +a=fmtp:109 apt=108 +a=rtpmap:127 H264/90000 +a=rtcp-fb:127 goog-remb +a=rtcp-fb:127 transport-cc +a=rtcp-fb:127 ccm fir +a=rtcp-fb:127 nack +a=rtcp-fb:127 nack pli +a=fmtp:127 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f +a=rtpmap:125 rtx/90000 +a=fmtp:125 apt=127 +a=rtpmap:39 H264/90000 +a=rtcp-fb:39 goog-remb +a=rtcp-fb:39 transport-cc +a=rtcp-fb:39 ccm fir +a=rtcp-fb:39 nack +a=rtcp-fb:39 nack pli +a=fmtp:39 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f +a=rtpmap:40 rtx/90000 +a=fmtp:40 apt=39 +a=rtpmap:45 AV1/90000 +a=rtcp-fb:45 goog-remb +a=rtcp-fb:45 transport-cc +a=rtcp-fb:45 ccm fir +a=rtcp-fb:45 nack +a=rtcp-fb:45 nack pli +a=rtpmap:46 rtx/90000 +a=fmtp:46 apt=45 +a=rtpmap:98 VP9/90000 +a=rtcp-fb:98 goog-remb +a=rtcp-fb:98 transport-cc +a=rtcp-fb:98 ccm fir +a=rtcp-fb:98 nack +a=rtcp-fb:98 nack pli +a=fmtp:98 profile-id=0 +a=rtpmap:99 rtx/90000 +a=fmtp:99 apt=98 +a=rtpmap:100 VP9/90000 +a=rtcp-fb:100 goog-remb +a=rtcp-fb:100 transport-cc +a=rtcp-fb:100 ccm fir +a=rtcp-fb:100 nack +a=rtcp-fb:100 nack pli +a=fmtp:100 profile-id=2 +a=rtpmap:101 rtx/90000 +a=fmtp:101 apt=100 +a=rtpmap:112 H264/90000 +a=rtcp-fb:112 goog-remb +a=rtcp-fb:112 transport-cc +a=rtcp-fb:112 ccm fir +a=rtcp-fb:112 nack +a=rtcp-fb:112 nack pli +a=fmtp:112 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f +a=rtpmap:113 rtx/90000 +a=fmtp:113 apt=112 +a=rtpmap:116 red/90000 +a=rtpmap:117 rtx/90000 +a=fmtp:117 apt=116 +a=rtpmap:118 ulpfec/90000 +a=ssrc-group:FID 4072017687 3466187747 +a=ssrc:4072017687 cname:mbGW+aeWMLFhPbBC +a=ssrc:4072017687 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +a=ssrc:3466187747 cname:mbGW+aeWMLFhPbBC +a=ssrc:3466187747 msid:04f96a2a-d107-4a12-b545-cfe1dc2c4d37 4a1ae8a7-6c89-44d3-a9a1-02abce0012c7 +m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:GM64 +a=ice-pwd:ANZFilRRlZJ3bg9AD40eRu7n +a=ice-options:trickle +a=fingerprint:sha-256 38:1F:02:E5:2A:49:9A:2A:D9:8E:B9:9B:4C:40:21:B7:F1:C4:27:8E:B5:68:D6:E0:91:08:D9:CB:2B:AC:B3:87 +a=setup:actpass +a=mid:3 +a=extmap:14 urn:ietf:params:rtp-hdrext:ssrc-audio-level +a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time +a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 +a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a +a=rtcp-mux +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;usedtx=1;useinbandfec=1 +a=rtpmap:63 red/48000/2 +a=fmtp:63 111/111 +a=rtpmap:9 G722/8000 +a=rtpmap:0 PCMU/8000 +a=rtpmap:8 PCMA/8000 +a=rtpmap:13 CN/8000 +a=rtpmap:110 telephone-event/48000 +a=rtpmap:126 telephone-event/8000 +a=ssrc:1281279951 cname:mbGW+aeWMLFhPbBC +a=ssrc:1281279951 msid:- 75b8c290-1f46-464c-8dc5-a4a3bced313a +`.trim(); diff --git a/packages/client/src/helpers/__tests__/sdp-munging.test.ts b/packages/client/src/helpers/__tests__/sdp-munging.test.ts index 22e6d58954..186c14183b 100644 --- a/packages/client/src/helpers/__tests__/sdp-munging.test.ts +++ b/packages/client/src/helpers/__tests__/sdp-munging.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { setPreferredCodec, removeCodec, toggleDtx } from '../sdp-munging'; +import { + enableHighQualityAudio, + removeCodec, + setPreferredCodec, + toggleDtx, +} from '../sdp-munging'; const sdpWithRed = `v=0 o=- 3265541491372987511 2 IN IP4 127.0.0.1 @@ -275,4 +280,12 @@ a=maxptime:40`; const dtxDisabledSdp = toggleDtx(dtxEnabledSdp, false); expect(dtxDisabledSdp.search('usedtx=0') !== -1).toBeTruthy(); }); + + it('enables HighQuality audio for Opus', async () => { + const { initialSdp: sdp } = await import('./hq-audio-sdp'); + + const sdpWithHighQualityAudio = enableHighQualityAudio(sdp, '3'); + expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000'); + expect(sdpWithHighQualityAudio).toContain('stereo=1'); + }); }); diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index abee224fd9..6af1af7fae 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -1,3 +1,5 @@ +import * as SDP from 'sdp-transform'; + type Media = { original: string; mediaWithPorts: string; @@ -227,3 +229,50 @@ export const toggleDtx = (sdp: string, enable: boolean): string => { } return sdp; }; + +/** + * Enables high-quality audio through SDP munging for the given trackMid. + * + * @param sdp the SDP to munge. + * @param trackMid the trackMid. + * @param maxBitrate the max bitrate to set. + */ +export const enableHighQualityAudio = ( + sdp: string, + trackMid: string, + maxBitrate: number = 510000, +): string => { + maxBitrate = Math.max(Math.min(maxBitrate, 510000), 96000); + + const parsedSdp = SDP.parse(sdp); + const audioMedia = parsedSdp.media.find( + (m) => m.type === 'audio' && String(m.mid) === trackMid, + ); + + if (!audioMedia) return sdp; + + const opusRtp = audioMedia.rtp.find((r) => r.codec === 'opus'); + if (!opusRtp) return sdp; + + const opusFmtp = audioMedia.fmtp.find((f) => f.payload === opusRtp.payload); + if (!opusFmtp) return sdp; + + // enable stereo, if not already enabled + if (opusFmtp.config.match(/stereo=(\d)/)) { + opusFmtp.config = opusFmtp.config.replace(/stereo=(\d)/, 'stereo=1'); + } else { + opusFmtp.config = `${opusFmtp.config};stereo=1`; + } + + // set maxaveragebitrate, to the given value + if (opusFmtp.config.match(/maxaveragebitrate=(\d*)/)) { + opusFmtp.config = opusFmtp.config.replace( + /maxaveragebitrate=(\d*)/, + `maxaveragebitrate=${maxBitrate}`, + ); + } else { + opusFmtp.config = `${opusFmtp.config};maxaveragebitrate=${maxBitrate}`; + } + + return SDP.write(parsedSdp); +}; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 4d8b370b38..0518ad2cd9 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -21,7 +21,7 @@ import { import { CallState } from '../store'; import { PublishOptions } from '../types'; import { isReactNative } from '../helpers/platforms'; -import { toggleDtx } from '../helpers/sdp-munging'; +import { enableHighQualityAudio, toggleDtx } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; @@ -71,7 +71,7 @@ export class Publisher { [TrackType.AUDIO]: 'audio', [TrackType.VIDEO]: 'video', [TrackType.SCREEN_SHARE]: 'video', - [TrackType.SCREEN_SHARE_AUDIO]: undefined, + [TrackType.SCREEN_SHARE_AUDIO]: 'audio', [TrackType.UNSPECIFIED]: undefined, }; @@ -535,7 +535,24 @@ export class Publisher { this.isIceRestarting = options?.iceRestart ?? false; const offer = await this.pc.createOffer(options); - offer.sdp = this.mungeCodecs(offer.sdp); + let sdp = this.mungeCodecs(offer.sdp); + if (sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + const transceiver = + this.transceiverRegistry[TrackType.SCREEN_SHARE_AUDIO]; + if (transceiver && transceiver.sender.track) { + const mid = + transceiver.mid ?? + this.extractMid( + sdp, + transceiver.sender.track, + TrackType.SCREEN_SHARE_AUDIO, + ); + sdp = enableHighQualityAudio(sdp, mid); + } + } + + // set the munged SDP back to the offer + offer.sdp = sdp; const trackInfos = this.getCurrentTrackInfos(offer.sdp); if (trackInfos.length === 0) { @@ -584,48 +601,48 @@ export class Publisher { return sdp; }; - getCurrentTrackInfos = (sdp?: string) => { - sdp = sdp || this.pc.localDescription?.sdp; - const extractMid = ( - defaultMid: string | null, - track: MediaStreamTrack, - trackType: TrackType, - ): string => { - if (defaultMid) return defaultMid; - if (!sdp) { - logger('warn', 'No SDP found. Returning empty mid'); - return ''; - } + private extractMid = ( + sdp: string | undefined, + track: MediaStreamTrack, + trackType: TrackType, + ): string => { + if (!sdp) { + logger('warn', 'No SDP found. Returning empty mid'); + return ''; + } + + logger( + 'debug', + `No 'mid' found for track. Trying to find it from the Offer SDP`, + ); + const parsedSdp = SDP.parse(sdp); + const media = parsedSdp.media.find((m) => { + return ( + m.type === track.kind && + // if `msid` is not present, we assume that the track is the first one + (m.msid?.includes(track.id) ?? true) + ); + }); + if (typeof media?.mid === 'undefined') { logger( 'debug', - `No 'mid' found for track. Trying to find it from the Offer SDP`, + `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`, ); - const parsedSdp = SDP.parse(sdp); - const media = parsedSdp.media.find((m) => { - return ( - m.type === track.kind && - // if `msid` is not present, we assume that the track is the first one - (m.msid?.includes(track.id) ?? true) - ); - }); - if (typeof media?.mid === 'undefined') { - logger( - 'debug', - `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`, - ); + const heuristicMid = this.transceiverInitOrder.indexOf(trackType); + if (heuristicMid !== -1) { + return String(heuristicMid); + } - const heuristicMid = this.transceiverInitOrder.indexOf(trackType); - if (heuristicMid !== -1) { - return String(heuristicMid); - } + logger('debug', 'No heuristic mid found. Returning empty mid'); + return ''; + } + return String(media.mid); + }; - logger('debug', 'No heuristic mid found. Returning empty mid'); - return ''; - } - return String(media.mid); - }; + getCurrentTrackInfos = (sdp?: string) => { + sdp = sdp || this.pc.localDescription?.sdp; const { settings } = this.state; const targetResolution = settings?.video.target_resolution; @@ -670,16 +687,24 @@ export class Publisher { }, })); + const isAudioTrack = [ + TrackType.AUDIO, + TrackType.SCREEN_SHARE_AUDIO, + ].includes(trackType); + + const trackSettings = track.getSettings(); + // @ts-expect-error - `channelCount` is not defined on `MediaTrackSettings` + const isStereo = isAudioTrack && trackSettings.channelCount === 2; + return { trackId: track.id, layers: layers, trackType, - mid: extractMid(transceiver.mid, track, trackType), + mid: transceiver.mid ?? this.extractMid(sdp, track, trackType), - // FIXME OL: adjust these values - stereo: false, - dtx: TrackType.AUDIO === trackType && this.isDtxEnabled, - red: TrackType.AUDIO === trackType && this.isRedEnabled, + stereo: isStereo, + dtx: isAudioTrack && this.isDtxEnabled, + red: isAudioTrack && this.isRedEnabled, }; }); }; diff --git a/packages/client/src/rtc/Subscriber.ts b/packages/client/src/rtc/Subscriber.ts index 3c16412a84..e4bf6e21e4 100644 --- a/packages/client/src/rtc/Subscriber.ts +++ b/packages/client/src/rtc/Subscriber.ts @@ -264,6 +264,7 @@ export class Subscriber { TRACK_TYPE_AUDIO: 'audioStream', TRACK_TYPE_VIDEO: 'videoStream', TRACK_TYPE_SCREEN_SHARE: 'screenShareStream', + TRACK_TYPE_SCREEN_SHARE_AUDIO: 'screenShareAudioStream', } as const )[trackType]; @@ -320,6 +321,11 @@ export class Subscriber { ); const answer = await this.pc.createAnswer(); + // FIXME OL: make this resilient, and apply to ScreenShare Audio tracks + answer.sdp = answer.sdp!.replace( + 'useinbandfec=1', + 'useinbandfec=1; stereo=1; maxaveragebitrate=510000', + ); await this.pc.setLocalDescription(answer); await this.sfuClient.sendAnswer({ diff --git a/packages/client/src/rtc/helpers/tracks.ts b/packages/client/src/rtc/helpers/tracks.ts index ae7d4ef1ab..4791711ce5 100644 --- a/packages/client/src/rtc/helpers/tracks.ts +++ b/packages/client/src/rtc/helpers/tracks.ts @@ -3,6 +3,7 @@ import type { StreamVideoLocalParticipant, StreamVideoParticipant, } from '../../types'; +import { TrackMuteType } from '../../types'; export const trackTypeToParticipantStreamKey = ( trackType: TrackType, @@ -10,12 +11,17 @@ export const trackTypeToParticipantStreamKey = ( switch (trackType) { case TrackType.SCREEN_SHARE: return 'screenShareStream'; + case TrackType.SCREEN_SHARE_AUDIO: + return 'screenShareAudioStream'; case TrackType.VIDEO: return 'videoStream'; case TrackType.AUDIO: return 'audioStream'; + case TrackType.UNSPECIFIED: + throw new Error('Track type is unspecified'); default: - throw new Error(`Unknown track type: ${trackType}`); + const exhaustiveTrackTypeCheck: never = trackType; + throw new Error(`Unknown track type: ${exhaustiveTrackTypeCheck}`); } }; @@ -28,15 +34,16 @@ export const trackTypeToDeviceIdKey = ( case TrackType.VIDEO: return 'videoDeviceId'; case TrackType.SCREEN_SHARE: + case TrackType.SCREEN_SHARE_AUDIO: + case TrackType.UNSPECIFIED: return undefined; default: - throw new Error(`Unknown track type: ${trackType}`); + const exhaustiveTrackTypeCheck: never = trackType; + throw new Error(`Unknown track type: ${exhaustiveTrackTypeCheck}`); } }; -export const muteTypeToTrackType = ( - muteType: 'audio' | 'video' | 'screenshare', -): TrackType => { +export const muteTypeToTrackType = (muteType: TrackMuteType): TrackType => { switch (muteType) { case 'audio': return TrackType.AUDIO; @@ -44,7 +51,10 @@ export const muteTypeToTrackType = ( return TrackType.VIDEO; case 'screenshare': return TrackType.SCREEN_SHARE; + case 'screenshare_audio': + return TrackType.SCREEN_SHARE_AUDIO; default: - throw new Error(`Unknown mute type: ${muteType}`); + const exhaustiveMuteTypeCheck: never = muteType; + throw new Error(`Unknown mute type: ${exhaustiveMuteTypeCheck}`); } }; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 2c487042cf..9b974b1e20 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -50,6 +50,12 @@ export interface StreamVideoParticipant extends Participant { */ screenShareStream?: MediaStream; + /** + * The participant's screen audio stream, if they are sharing their audio, + * and we are subscribed to it. + */ + screenShareAudioStream?: MediaStream; + /** * The preferred video dimensions for this participant. * Set it to `undefined` to unsubscribe from this participant's video. @@ -110,6 +116,12 @@ export interface StreamVideoLocalParticipant extends StreamVideoParticipant { } export type VideoTrackType = 'videoTrack' | 'screenShareTrack'; +export type AudioTrackType = 'audioTrack' | 'screenShareAudioTrack'; +export type TrackMuteType = + | 'audio' + | 'video' + | 'screenshare' + | 'screenshare_audio'; /** * Represents a participant's pin state. diff --git a/packages/react-sdk/src/core/components/Audio/Audio.tsx b/packages/react-sdk/src/core/components/Audio/Audio.tsx index da16da6cc9..ae795edde4 100644 --- a/packages/react-sdk/src/core/components/Audio/Audio.tsx +++ b/packages/react-sdk/src/core/components/Audio/Audio.tsx @@ -1,12 +1,28 @@ import { ComponentPropsWithoutRef, useEffect, useState } from 'react'; -import { StreamVideoParticipant } from '@stream-io/video-client'; +import { + AudioTrackType, + StreamVideoParticipant, +} from '@stream-io/video-client'; import { useCall } from '@stream-io/video-react-bindings'; export type AudioProps = ComponentPropsWithoutRef<'audio'> & { + /** + * The participant whose audio stream we want to play. + */ participant: StreamVideoParticipant; + + /** + * The type of audio stream to play for the given participant. + * The default value is `audioTrack`. + */ + trackType?: AudioTrackType; }; -export const Audio = ({ participant, ...rest }: AudioProps) => { +export const Audio = ({ + participant, + trackType = 'audioTrack', + ...rest +}: AudioProps) => { const call = useCall(); const [audioElement, setAudioElement] = useState( null, @@ -15,13 +31,11 @@ export const Audio = ({ participant, ...rest }: AudioProps) => { useEffect(() => { if (!call || !audioElement) return; - - const cleanup = call.bindAudioElement(audioElement, sessionId); - + const cleanup = call.bindAudioElement(audioElement, sessionId, trackType); return () => { cleanup?.(); }; - }, [call, sessionId, audioElement]); + }, [call, sessionId, audioElement, trackType]); return (