diff --git a/packages/hms_room_kit/lib/src/assets/icons/cc-filled.svg b/packages/hms_room_kit/lib/src/assets/icons/cc-filled.svg new file mode 100644 index 000000000..3e3dd9464 --- /dev/null +++ b/packages/hms_room_kit/lib/src/assets/icons/cc-filled.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/hms_room_kit/lib/src/assets/icons/cc.svg b/packages/hms_room_kit/lib/src/assets/icons/cc.svg new file mode 100644 index 000000000..cf20fa13f --- /dev/null +++ b/packages/hms_room_kit/lib/src/assets/icons/cc.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/hms_room_kit/lib/src/meeting/meeting_page.dart b/packages/hms_room_kit/lib/src/meeting/meeting_page.dart index 3a422284a..3d5dfc85b 100644 --- a/packages/hms_room_kit/lib/src/meeting/meeting_page.dart +++ b/packages/hms_room_kit/lib/src/meeting/meeting_page.dart @@ -26,6 +26,7 @@ import 'package:hms_room_kit/src/preview_for_role/preview_for_role_bottom_sheet. import 'package:hms_room_kit/src/preview_for_role/preview_for_role_header.dart'; import 'package:hms_room_kit/src/widgets/common_widgets/hms_circular_avatar.dart'; import 'package:hms_room_kit/src/widgets/common_widgets/hms_left_room_screen.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/transcription_view.dart'; ///[MeetingPage] is the main page of the meeting ///It takes the following parameters: @@ -152,6 +153,10 @@ class _MeetingPageState extends State { ], ), + ChangeNotifierProvider.value( + value: _visibilityController, + child: const TranscriptionView()), + ///This gets rendered when the previewForRole method is called ///This is used to show the preview for role component Selector< diff --git a/packages/hms_room_kit/lib/src/meeting/meeting_store.dart b/packages/hms_room_kit/lib/src/meeting/meeting_store.dart index 64fa171d2..adcf9a6eb 100644 --- a/packages/hms_room_kit/lib/src/meeting/meeting_store.dart +++ b/packages/hms_room_kit/lib/src/meeting/meeting_store.dart @@ -1,6 +1,7 @@ ///Dart imports library; +import 'dart:async'; import 'dart:convert'; import 'dart:developer'; import 'dart:io'; @@ -38,7 +39,8 @@ class MeetingStore extends ChangeNotifier HMSKeyChangeListener, HMSHLSPlaybackEventsListener, HMSPollListener, - HMSWhiteboardUpdateListener { + HMSWhiteboardUpdateListener, + HMSTranscriptListener { late HMSSDKInteractor _hmsSDKInteractor; MeetingStore({required HMSSDKInteractor hmsSDKInteractor}) { @@ -276,6 +278,13 @@ class MeetingStore extends ChangeNotifier ///variable to store whiteboard model HMSWhiteboardModel? whiteboardModel; + ///variable to store whether transcription is enabled or not + bool isTranscriptionEnabled = false; + + bool isTranscriptionDisplayed = false; + + List captions = []; + Future join(String userName, String? tokenData) async { late HMSConfig joinConfig; @@ -475,6 +484,9 @@ class MeetingStore extends ChangeNotifier case HMSToastsType.streamingErrorToast: toasts.removeWhere( (toast) => toast.hmsToastType == HMSToastsType.streamingErrorToast); + case HMSToastsType.transcriptionToast: + toasts.removeWhere( + (toast) => toast.hmsToastType == HMSToastsType.transcriptionToast); } notifyListeners(); } @@ -824,6 +836,40 @@ class MeetingStore extends ChangeNotifier notifyListeners(); } + ///This method is used to toggle the transcription + ///for the peer who has admin permissions + void toggleTranscription() async { + HMSException? result; + toasts.add(HMSToastModel( + isTranscriptionEnabled + ? "Disabling Closed Captioning for everyone" + : "Enabling Closed Captioning for everyone", + hmsToastType: HMSToastsType.transcriptionToast)); + if (isTranscriptionEnabled) { + result = await HMSTranscriptionController.stopTranscription(); + } else { + result = await HMSTranscriptionController.startTranscription(); + } + if (result == null) { + isTranscriptionEnabled = !isTranscriptionEnabled; + toggleTranscriptionDisplay(); + } else { + removeToast(HMSToastsType.transcriptionToast); + toasts.add(HMSToastModel(result, hmsToastType: HMSToastsType.errorToast)); + } + notifyListeners(); + } + + void toggleTranscriptionDisplay() { + isTranscriptionDisplayed = !isTranscriptionDisplayed; + if (isTranscriptionDisplayed) { + HMSTranscriptionController.addListener(listener: this); + } else { + HMSTranscriptionController.removeListener(); + } + notifyListeners(); + } + // Override Methods @override @@ -850,6 +896,16 @@ class MeetingStore extends ChangeNotifier streamingType["hls"] = room.hmshlsStreamingState?.state ?? HMSStreamingState.none; + int index = room.transcriptions?.indexWhere( + (element) => element.mode == HMSTranscriptionMode.caption) ?? + -1; + + if (index != -1) { + room.transcriptions?[index].state == HMSTranscriptionState.started + ? isTranscriptionEnabled = true + : isTranscriptionEnabled = false; + } + checkNoiseCancellationAvailability(); setParticipantsList(roles); toggleAlwaysScreenOn(); @@ -906,6 +962,7 @@ class MeetingStore extends ChangeNotifier notifyListeners(); fetchPollList(HMSPollState.stopped); HMSWhiteboardController.addHMSWhiteboardUpdateListener(listener: this); + HMSTranscriptionController.addListener(listener: this); if (HMSRoomLayout.roleLayoutData?.screens?.preview?.joinForm?.joinBtnType == JoinButtonType.JOIN_BTN_TYPE_JOIN_AND_GO_LIVE && @@ -1000,6 +1057,28 @@ class MeetingStore extends ChangeNotifier ? room.hmshlsStreamingState?.variants[0]?.hlsStreamUrl : null; break; + case HMSRoomUpdate.transcriptionsUpdated: + if (room.transcriptions?.isNotEmpty ?? false) { + int index = room.transcriptions?.indexWhere( + (element) => element.mode == HMSTranscriptionMode.caption) ?? + -1; + + if (index != -1) { + if (room.transcriptions?[index].state == + HMSTranscriptionState.started || + room.transcriptions?[index].state == + HMSTranscriptionState.stopped) { + removeToast(HMSToastsType.transcriptionToast); + } + if (room.transcriptions?[index].state == + HMSTranscriptionState.started) { + isTranscriptionEnabled = true; + } else { + isTranscriptionEnabled = false; + } + } + } + break; default: break; } @@ -1406,6 +1485,7 @@ class MeetingStore extends ChangeNotifier HMSHLSPlayerController.removeHMSHLSPlaybackEventsListener(this); HMSPollInteractivityCenter.removePollUpdateListener(); HMSWhiteboardController.removeHMSWhiteboardUpdateListener(); + HMSTranscriptionController.removeListener(); } ///Function to toggle screen share @@ -2676,6 +2756,29 @@ class MeetingStore extends ChangeNotifier notifyListeners(); } + bool areCaptionsEmpty = true; + Timer? transcriptionTimerObj; + + @override + void onTranscripts({required List transcriptions}) { + areCaptionsEmpty = false; + captions = transcriptions; + startTranscriptionHideTimer(); + transcriptions.forEach((element) { + log("onTranscripts -> text: ${element.transcript}"); + }); + notifyListeners(); + } + + void startTranscriptionHideTimer() { + transcriptionTimerObj?.cancel(); + transcriptionTimerObj = Timer(Duration(seconds: 4), () { + areCaptionsEmpty = true; + captions = []; + notifyListeners(); + }); + } + //Get onSuccess or onException callbacks for HMSActionResultListenerMethod @override void onSuccess( diff --git a/packages/hms_room_kit/lib/src/widgets/bottom_sheets/app_utilities_bottom_sheet.dart b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/app_utilities_bottom_sheet.dart index 5177fe126..ab5b3abcd 100644 --- a/packages/hms_room_kit/lib/src/widgets/bottom_sheets/app_utilities_bottom_sheet.dart +++ b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/app_utilities_bottom_sheet.dart @@ -4,6 +4,8 @@ library; import 'package:badges/badges.dart' as badge; import 'package:flutter/material.dart'; import 'package:flutter_svg/flutter_svg.dart'; +import 'package:hms_room_kit/src/widgets/bottom_sheets/closed_caption_bottom_sheet.dart'; +import 'package:hms_room_kit/src/widgets/bottom_sheets/closed_caption_control_bottom_sheet.dart'; import 'package:hmssdk_flutter/hmssdk_flutter.dart'; import 'package:provider/provider.dart'; @@ -42,7 +44,7 @@ class _AppUtilitiesBottomSheetState extends State { super.deactivate(); } - Color geWhiteboardStatusColor(MeetingStore meetingStore) { + Color getWhiteboardStatusColor(MeetingStore meetingStore) { ///If whiteboard is enabled and the local peer is the owner of the whiteboard ///we return high emphasis color since the local peer can close the whiteboard ///else we return low emphasis color since local peer can't close the whiteboard @@ -59,6 +61,15 @@ class _AppUtilitiesBottomSheetState extends State { : HMSThemeColors.onSurfaceHighEmphasis; } + bool getTranscriptionPermission(MeetingStore meetingStore) { + int? index = meetingStore.localPeer?.role.permissions.transcription + ?.indexWhere((element) => element.mode == HMSTranscriptionMode.caption); + if (index == null || index == -1) { + return false; + } + return meetingStore.localPeer!.role.permissions.transcription![index].admin; + } + @override Widget build(BuildContext context) { MeetingStore meetingStore = context.read(); @@ -414,14 +425,90 @@ class _AppUtilitiesBottomSheetState extends State { height: 20, width: 20, colorFilter: ColorFilter.mode( - geWhiteboardStatusColor(meetingStore), + getWhiteboardStatusColor(meetingStore), BlendMode.srcIn), ), - optionTextColor: geWhiteboardStatusColor(meetingStore), + optionTextColor: getWhiteboardStatusColor(meetingStore), optionText: meetingStore.isWhiteboardEnabled ? "Close Whiteboard" : "Open Whiteboard"), + ///This renders the closed captions option + ///This option is only rendered if the local peer has the permission to + ///enable/disable transcription(will see the popup to enable caption) + ///else the local peer can only see the option to Show/Hide captions + if (getTranscriptionPermission(meetingStore) || + meetingStore.isTranscriptionEnabled) + MoreOptionItem( + onTap: () async { + Navigator.pop(context); + + ///If the local peer has the permission to enable/disable transcription + ///we show the popup to enable/disable transcription + ///else we call the method to show/hide captions + (getTranscriptionPermission(meetingStore)) + ? showModalBottomSheet( + isScrollControlled: true, + backgroundColor: HMSThemeColors.surfaceDim, + shape: const RoundedRectangleBorder( + borderRadius: BorderRadius.only( + topLeft: Radius.circular(16), + topRight: Radius.circular(16)), + ), + context: context, + builder: (ctx) => ChangeNotifierProvider.value( + value: meetingStore, + child: meetingStore.isTranscriptionEnabled + ? ClosedCaptionControlBottomSheet( + meetingStore: meetingStore, + ) + : ClosedCaptionBottomSheet( + onButtonPressed: () => meetingStore + .toggleTranscription(), + title: HMSTitleText( + text: + "Enable Closed Captions (CC) for this session?", + maxLines: 5, + textColor: HMSThemeColors + .onSecondaryHighEmphasis, + letterSpacing: 0.15, + fontSize: 20, + ), + subTitle: HMSSubheadingText( + text: + "This will enable Closed Captions for everyone in this room. You can disable it later.", + maxLines: 2, + textColor: HMSThemeColors + .onSurfaceMediumEmphasis, + ), + buttonText: "Enable for Everyone", + ), + ), + ) + : meetingStore.toggleTranscriptionDisplay(); + }, + + ///The button is active if the transcription is enabled and getting displayed + isActive: meetingStore.isTranscriptionDisplayed, + optionIcon: SvgPicture.asset( + "packages/hms_room_kit/lib/src/assets/icons/${meetingStore.isTranscriptionDisplayed ? "cc-filled" : "cc"}.svg", + height: 20, + width: 20, + colorFilter: ColorFilter.mode( + HMSThemeColors.onSurfaceHighEmphasis, + BlendMode.srcIn), + ), + optionTextColor: HMSThemeColors.onSurfaceHighEmphasis, + + ///If the local peer has the permission to enable/disable transcription + ///we show the option to enable/disable transcription + ///else we show the option to show/hide captions + optionText: getTranscriptionPermission(meetingStore) + ? "Closed Captions" + : meetingStore.isTranscriptionDisplayed + ? "Hide Captions" + : "Show Captions"), + ///Virtual background is not supported out of the box in prebuilt as of now // if (AppDebugConfig.isVirtualBackgroundEnabled && // (meetingStore.localPeer?.role.publishSettings?.allowed diff --git a/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_bottom_sheet.dart b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_bottom_sheet.dart new file mode 100644 index 000000000..7a5fc033d --- /dev/null +++ b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_bottom_sheet.dart @@ -0,0 +1,128 @@ +///Package imports +library; + +import 'package:flutter/material.dart'; + +///Project imports +import 'package:hms_room_kit/src/layout_api/hms_theme_colors.dart'; +import 'package:hms_room_kit/src/meeting/meeting_store.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/hms_cross_button.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/hms_title_text.dart'; +import 'package:provider/provider.dart'; + +///[EndServiceBottomSheet] is a bottom sheet that is used to render the bottom sheet to stop services +///It has following parameters: +///[bottomSheetTitleIcon] is the icon that is shown on the top left of the bottom sheet +/// [title] is the title of the bottom sheet +/// [subTitle] is the subtitle of the bottom sheet +/// [buttonText] is the text of the button +/// [onButtonPressed] is the function that is called when the button is pressed +/// [buttonColor] is the color of the button +class ClosedCaptionBottomSheet extends StatefulWidget { + final Widget? bottomSheetTitleIcon; + final Widget? title; + final Widget? subTitle; + final String? buttonText; + final Function? onButtonPressed; + final Color? buttonColor; + + const ClosedCaptionBottomSheet( + {super.key, + this.bottomSheetTitleIcon, + this.title, + this.subTitle, + this.buttonText, + this.onButtonPressed, + this.buttonColor}); + + @override + State createState() => + _ClosedCaptionBottomSheetState(); +} + +class _ClosedCaptionBottomSheetState extends State { + @override + void initState() { + super.initState(); + context.read().addBottomSheet(context); + } + + @override + void deactivate() { + context.read().removeBottomSheet(context); + super.deactivate(); + } + + @override + Widget build(BuildContext context) { + return FractionallySizedBox( + heightFactor: MediaQuery.of(context).orientation == Orientation.portrait + ? 0.30 + : 0.45, + child: Padding( + padding: const EdgeInsets.only(top: 16.0, left: 20, right: 20), + child: SingleChildScrollView( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Row( + children: [ + widget.bottomSheetTitleIcon ?? const SizedBox(), + widget.bottomSheetTitleIcon ?? + const SizedBox( + width: 8, + ), + SizedBox( + width: MediaQuery.of(context).size.width - 100, + child: widget.title ?? const SizedBox()) + ], + ), + const Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + HMSCrossButton(), + ], + ) + ], + ), + const SizedBox( + height: 16, + ), + ElevatedButton( + style: ButtonStyle( + shadowColor: + MaterialStateProperty.all(HMSThemeColors.surfaceDim), + backgroundColor: MaterialStateProperty.all( + widget.buttonColor ?? HMSThemeColors.primaryDefault), + shape: MaterialStateProperty.all( + RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.0), + ))), + onPressed: () { + if (widget.onButtonPressed != null) { + widget.onButtonPressed!(); + } + Navigator.pop(context); + }, + child: SizedBox( + height: 48, + child: Center( + child: HMSTitleText( + text: widget.buttonText ?? "", + textColor: HMSThemeColors.onSurfaceHighEmphasis), + ), + )), + const SizedBox( + height: 16, + ), + widget.subTitle ?? const SizedBox(), + ], + ), + ), + ), + ); + } +} diff --git a/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_control_bottom_sheet.dart b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_control_bottom_sheet.dart new file mode 100644 index 000000000..c589f36db --- /dev/null +++ b/packages/hms_room_kit/lib/src/widgets/bottom_sheets/closed_caption_control_bottom_sheet.dart @@ -0,0 +1,132 @@ +import 'package:flutter/material.dart'; +import 'package:hms_room_kit/hms_room_kit.dart'; +import 'package:hms_room_kit/src/meeting/meeting_store.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/hms_cross_button.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/hms_subheading_text.dart'; +import 'package:provider/provider.dart'; + +class ClosedCaptionControlBottomSheet extends StatefulWidget { + final MeetingStore meetingStore; + + const ClosedCaptionControlBottomSheet( + {super.key, required this.meetingStore}); + + @override + State createState() => + _ClosedCaptionControlBottomSheetState(); +} + +class _ClosedCaptionControlBottomSheetState + extends State { + @override + void initState() { + super.initState(); + context.read().addBottomSheet(context); + } + + @override + void deactivate() { + context.read().removeBottomSheet(context); + super.deactivate(); + } + + @override + Widget build(BuildContext context) { + return FractionallySizedBox( + heightFactor: MediaQuery.of(context).orientation == Orientation.portrait + ? 0.37 + : 0.45, + child: Padding( + padding: const EdgeInsets.only(top: 16.0, left: 20, right: 20), + child: SingleChildScrollView( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Row( + children: [ + HMSTitleText( + text: "Closed Captions (CC)", + textColor: HMSThemeColors.onSecondaryHighEmphasis, + letterSpacing: 0.15, + fontSize: 20, + ), + ], + ), + Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + const HMSCrossButton(), + ], + ) + ], + ), + const SizedBox( + height: 16, + ), + ElevatedButton( + style: ButtonStyle( + shadowColor: + MaterialStateProperty.all(HMSThemeColors.surfaceDim), + backgroundColor: MaterialStateProperty.all( + HMSThemeColors.secondaryDefault), + shape: MaterialStateProperty.all( + RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.0), + ))), + onPressed: () { + widget.meetingStore.toggleTranscriptionDisplay(); + Navigator.pop(context); + }, + child: SizedBox( + height: 48, + child: Center( + child: HMSTitleText( + text: + "${widget.meetingStore.isTranscriptionDisplayed ? "Hide" : "Show"} for Me", + textColor: HMSThemeColors.onSecondaryHighEmphasis), + ), + )), + const SizedBox( + height: 16, + ), + ElevatedButton( + style: ButtonStyle( + shadowColor: + MaterialStateProperty.all(HMSThemeColors.surfaceDim), + backgroundColor: MaterialStateProperty.all( + HMSThemeColors.alertErrorDefault), + shape: MaterialStateProperty.all( + RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8.0), + ))), + onPressed: () { + widget.meetingStore.toggleTranscription(); + Navigator.pop(context); + }, + child: SizedBox( + height: 48, + child: Center( + child: HMSTitleText( + text: "Disable For Everyone", + textColor: HMSThemeColors.alertErrorBrighter), + ), + )), + const SizedBox( + height: 16, + ), + HMSSubheadingText( + text: + "This will disable Closed Captions for everyone in this room. You can enable it again.", + maxLines: 2, + textColor: HMSThemeColors.onSurfaceMediumEmphasis, + ), + ], + ), + ), + ), + ); + } +} diff --git a/packages/hms_room_kit/lib/src/widgets/common_widgets/transcription_view.dart b/packages/hms_room_kit/lib/src/widgets/common_widgets/transcription_view.dart new file mode 100644 index 000000000..5d3662e55 --- /dev/null +++ b/packages/hms_room_kit/lib/src/widgets/common_widgets/transcription_view.dart @@ -0,0 +1,106 @@ +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:hms_room_kit/hms_room_kit.dart'; +import 'package:hms_room_kit/src/meeting/meeting_navigation_visibility_controller.dart'; +import 'package:hms_room_kit/src/meeting/meeting_store.dart'; +import 'package:hmssdk_flutter/hmssdk_flutter.dart'; +import 'package:provider/provider.dart'; +import 'package:tuple/tuple.dart'; + +class TranscriptionView extends StatefulWidget { + const TranscriptionView({Key? key}) : super(key: key); + @override + State createState() => _TranscriptionViewState(); +} + +class _TranscriptionViewState extends State { + @override + Widget build(BuildContext context) { + return Selector( + selector: (_, meetingStore) => meetingStore.isTranscriptionDisplayed, + builder: (_, isTranscriptionDisplayed, __) { + return isTranscriptionDisplayed + ? Selector, int>>( + selector: (_, meetingStore) => Tuple2( + meetingStore.captions, meetingStore.captions.length), + builder: (_, data, __) { + String transcript = ""; + for (var i = 0; i < data.item2; i++) { + transcript += + "${data.item1[i].peerName}: ${data.item1[i].transcript}\n"; + } + return data.item2 > 0 + ? Selector>( + selector: (_, meetingStore) => Tuple2( + meetingStore.isOverlayChatOpened, + meetingStore.toasts.isNotEmpty), + builder: (_, isUIElementPresent, __) { + return Selector< + MeetingNavigationVisibilityController, + bool>( + selector: + (_, meetingNavigationVisibilityController) => + meetingNavigationVisibilityController + .showControls, + builder: (_, showControls, __) { + double bottomMargin = showControls + ? Platform.isIOS + ? 110 + : 90 + : Platform.isIOS + ? 80 + : 50; + + ///If toasts are present then we need to adjust the bottom margin + bottomMargin += isUIElementPresent.item2 + ? showControls + ? 40 + : 80 + : 0; + return Positioned( + ///If overlay chat is opened then we need to adjust the top margin(shift the chat to top) + ///else we keep it at the bottom + bottom: isUIElementPresent.item1 + ? null + : bottomMargin, + top: isUIElementPresent.item1 + ? showControls + ? 80 + : 50 + : null, + child: Column( + mainAxisSize: MainAxisSize.min, + mainAxisAlignment: + MainAxisAlignment.center, + children: [ + Container( + width: MediaQuery.of(context) + .size + .width - + 10, + padding: const EdgeInsets.all(5), + margin: const EdgeInsets.all(2), + decoration: BoxDecoration( + // color: Colors.red, + color: Colors.black + .withAlpha(64), + borderRadius: + const BorderRadius.all( + Radius.circular(10))), + child: HMSSubtitleText( + text: transcript, + maxLines: 5, + textColor: HMSThemeColors + .onSurfaceHighEmphasis)), + ], + ), + ); + }); + }) + : const SizedBox(); + }) + : const SizedBox(); + }); + } +} diff --git a/packages/hms_room_kit/lib/src/widgets/toasts/hms_toasts_type.dart b/packages/hms_room_kit/lib/src/widgets/toasts/hms_toasts_type.dart index d617daace..de30b4679 100644 --- a/packages/hms_room_kit/lib/src/widgets/toasts/hms_toasts_type.dart +++ b/packages/hms_room_kit/lib/src/widgets/toasts/hms_toasts_type.dart @@ -7,5 +7,6 @@ enum HMSToastsType { chatPauseResumeToast, errorToast, pollStartedToast, - streamingErrorToast + streamingErrorToast, + transcriptionToast } diff --git a/packages/hms_room_kit/lib/src/widgets/toasts/hms_transcription_toast.dart b/packages/hms_room_kit/lib/src/widgets/toasts/hms_transcription_toast.dart new file mode 100644 index 000000000..5ec2c9d7e --- /dev/null +++ b/packages/hms_room_kit/lib/src/widgets/toasts/hms_transcription_toast.dart @@ -0,0 +1,33 @@ +import 'package:flutter/material.dart'; +import 'package:hms_room_kit/hms_room_kit.dart'; +import 'package:hms_room_kit/src/meeting/meeting_store.dart'; +import 'package:hms_room_kit/src/widgets/common_widgets/hms_subheading_text.dart'; +import 'package:hms_room_kit/src/widgets/toasts/hms_toast.dart'; + +class HMSTranscriptionToast extends StatelessWidget { + final String message; + final MeetingStore meetingStore; + const HMSTranscriptionToast( + {Key? key, required this.message, required this.meetingStore}); + + @override + Widget build(BuildContext context) { + return HMSToast( + leading: SizedBox( + height: 18, + width: 18, + child: CircularProgressIndicator( + strokeWidth: 2, + color: HMSThemeColors.onSurfaceHighEmphasis, + ), + ), + subtitle: HMSSubheadingText( + text: message, + textColor: HMSThemeColors.onSurfaceHighEmphasis, + fontWeight: FontWeight.w600, + letterSpacing: 0.1, + textOverflow: TextOverflow.ellipsis, + maxLines: 2, + )); + } +} diff --git a/packages/hms_room_kit/lib/src/widgets/toasts/toast_widget.dart b/packages/hms_room_kit/lib/src/widgets/toasts/toast_widget.dart index 2ae64da65..1b8c6ecb0 100644 --- a/packages/hms_room_kit/lib/src/widgets/toasts/toast_widget.dart +++ b/packages/hms_room_kit/lib/src/widgets/toasts/toast_widget.dart @@ -2,6 +2,9 @@ library; import 'package:flutter/material.dart'; +import 'package:hms_room_kit/src/widgets/toasts/hms_error_toast.dart'; +import 'package:hms_room_kit/src/widgets/toasts/hms_streaming_error_toast.dart'; +import 'package:hmssdk_flutter/hmssdk_flutter.dart'; import 'package:provider/provider.dart'; ///Project imports @@ -16,6 +19,7 @@ import 'package:hms_room_kit/src/widgets/toasts/hms_recording_error_toast.dart'; import 'package:hms_room_kit/src/widgets/toasts/hms_role_change_decline_toast.dart'; import 'package:hms_room_kit/src/widgets/toasts/hms_toast_model.dart'; import 'package:hms_room_kit/src/widgets/toasts/hms_toasts_type.dart'; +import 'package:hms_room_kit/src/widgets/toasts/hms_transcription_toast.dart'; ///[ToastWidget] returns toast based on the toast type class ToastWidget extends StatelessWidget { @@ -74,8 +78,24 @@ class ToastWidget extends StatelessWidget { ), ), ); + case HMSToastsType.transcriptionToast: + return HMSTranscriptionToast( + message: toast.toastData, + meetingStore: meetingStore, + ); + // default: + // return const SizedBox(); + case HMSToastsType.errorToast: + if (toast.toastData is HMSException) { + return HMSErrorToast( + error: toast.toastData, meetingStore: meetingStore); + } + return SizedBox(); + case HMSToastsType.streamingErrorToast: + return HMSStreamingErrorToast( + streamingError: toast.toastData, meetingStore: meetingStore); default: - return const SizedBox(); + return SizedBox(); } } } diff --git a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSRoomExtension.kt b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSRoomExtension.kt index bebd5bd78..93b7b68b7 100644 --- a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSRoomExtension.kt +++ b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSRoomExtension.kt @@ -1,6 +1,7 @@ package live.hms.hmssdk_flutter import live.hms.video.sdk.models.HMSRoom +import live.hms.video.sdk.models.Transcriptions import live.hms.video.sdk.models.enums.HMSRoomUpdate class HMSRoomExtension { @@ -18,6 +19,7 @@ class HMSRoomExtension { args.add(HMSPeerExtension.toDictionary(it)!!) } + hashMap["transcriptions"] = HMSTranscriptExtension.getMapFromTranscriptionsList(room.transcriptions) hashMap["is_large"] = room.isLargeRoom hashMap["local_peer"] = HMSPeerExtension.toDictionary(room.localPeer) hashMap["peers"] = args @@ -44,6 +46,7 @@ class HMSRoomExtension { HMSRoomUpdate.BROWSER_RECORDING_STATE_UPDATED -> "browser_recording_state_updated" HMSRoomUpdate.HLS_RECORDING_STATE_UPDATED -> "hls_recording_state_updated" HMSRoomUpdate.ROOM_PEER_COUNT_UPDATED -> "room_peer_count_updated" + HMSRoomUpdate.TRANSCRIPTIONS_UPDATED -> "transcriptions_updated" else -> "defaultUpdate" } } diff --git a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSTranscriptExtension.kt b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSTranscriptExtension.kt new file mode 100644 index 000000000..f46d0e4db --- /dev/null +++ b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HMSTranscriptExtension.kt @@ -0,0 +1,92 @@ +package live.hms.hmssdk_flutter + +import live.hms.video.sdk.models.OnTranscriptionError +import live.hms.video.sdk.models.TranscriptionState +import live.hms.video.sdk.models.Transcriptions +import live.hms.video.sdk.models.TranscriptionsMode +import live.hms.video.sdk.transcripts.HmsTranscript + +class HMSTranscriptExtension { + + companion object{ + + fun toDictionary(hmsTranscript: HmsTranscript):HashMap{ + + val args = HashMap() + + args["start"] = hmsTranscript.start + args["end"] = hmsTranscript.end + args["transcript"] = hmsTranscript.transcript + args["peer_id"] = hmsTranscript.peerId + args["peer_name"] = hmsTranscript.peer?.name + args["is_final"] = hmsTranscript.isFinal + return args + } + + fun getMapFromTranscriptionsList(transcriptions : List): ArrayList>{ + + val transcriptionList = ArrayList>() + + transcriptions.forEach {_transcription -> + transcriptionList.add(getMapFromTranscription(_transcription)) + } + + return transcriptionList + } + + private fun getMapFromTranscription(transcription: Transcriptions): HashMap{ + + val args = HashMap() + + args["error"] = transcriptionErrorExtension(transcription.error) + args["started_at"] = transcription.startedAt + args["stopped_at"] = transcription.stoppedAt + args["updated_at"] = transcription.updatedAt + args["state"] = getStringFromTranscriptionState(transcription.state) + args["mode"] = getStringFromTranscriptionMode(transcription.mode) + + return args + } + + private fun transcriptionErrorExtension(onTranscriptionError: OnTranscriptionError?): HashMap?{ + onTranscriptionError?.let {_transcriptionError -> + + val args = HashMap() + + args["code"] = _transcriptionError.code + args["message"] = _transcriptionError.message + + return args + + }?: run { + return null + } + } + + private fun getStringFromTranscriptionState(transcriptionState: TranscriptionState?): String?{ + return when(transcriptionState){ + TranscriptionState.STARTED -> "started" + TranscriptionState.STOPPED -> "stopped" + TranscriptionState.INITIALIZED -> "initialized" + TranscriptionState.FAILED -> "failed" + else -> null + } + } + + fun getTranscriptionModeFromString(transcriptionMode: String): TranscriptionsMode?{ + return when(transcriptionMode){ + "caption" -> TranscriptionsMode.CAPTION + "live" -> TranscriptionsMode.LIVE + else -> null + } + } + + fun getStringFromTranscriptionMode(transcriptionMode: TranscriptionsMode?): String?{ + return when(transcriptionMode){ + TranscriptionsMode.CAPTION -> "caption" + TranscriptionsMode.LIVE -> "live" + else -> null + } + } + } +} diff --git a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HmssdkFlutterPlugin.kt b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HmssdkFlutterPlugin.kt index 73b8f7db8..3d7cb5cdd 100644 --- a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HmssdkFlutterPlugin.kt +++ b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/HmssdkFlutterPlugin.kt @@ -45,6 +45,7 @@ import live.hms.video.sdk.models.* import live.hms.video.sdk.models.enums.* import live.hms.video.sdk.models.role.HMSRole import live.hms.video.sdk.models.trackchangerequest.HMSChangeTrackStateRequest +import live.hms.video.sdk.transcripts.HmsTranscripts import live.hms.video.sessionstore.HMSKeyChangeListener import live.hms.video.sessionstore.HmsSessionStore import live.hms.video.signal.init.* @@ -68,6 +69,7 @@ class HmssdkFlutterPlugin : var hlsPlayerChannel: EventChannel? = null private var pollsEventChannel: EventChannel? = null private var whiteboardEventChannel: EventChannel? = null + private var transcriptionEventChannel: EventChannel? = null private var eventSink: EventChannel.EventSink? = null private var previewSink: EventChannel.EventSink? = null private var logsSink: EventChannel.EventSink? = null @@ -76,6 +78,7 @@ class HmssdkFlutterPlugin : var hlsPlayerSink: EventChannel.EventSink? = null private var pollsSink: EventChannel.EventSink? = null private var whiteboardSink: EventChannel.EventSink? = null + private var transcriptionSink : EventChannel.EventSink? = null private lateinit var activity: Activity var hmssdk: HMSSDK? = null private lateinit var hmsVideoFactory: HMSVideoViewFactory @@ -121,6 +124,9 @@ class HmssdkFlutterPlugin : this.whiteboardEventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "whiteboard_event_channel") + this.transcriptionEventChannel = + EventChannel(flutterPluginBinding.binaryMessenger, "transcription_event_channel") + this.meetingEventChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "Meeting event channel not found") this.channel?.setMethodCallHandler(this) ?: Log.e("Channel Error", "Event channel not found") this.previewChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "Preview channel not found") @@ -130,6 +136,7 @@ class HmssdkFlutterPlugin : this.hlsPlayerChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "HLS Player channel not found") this.pollsEventChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "polls events channel not found") this.whiteboardEventChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "whiteboard events channel not found") + this.transcriptionEventChannel?.setStreamHandler(this) ?: Log.e("Channel Error", "transcription event channel not found") this.hmsVideoFactory = HMSVideoViewFactory(this) this.hmsHLSPlayerFactory = HMSHLSPlayerFactory(this) @@ -305,6 +312,10 @@ class HmssdkFlutterPlugin : whiteboardActions(call, result) } + "start_real_time_transcription", "stop_real_time_transcription","add_transcript_listener", "remove_transcript_listener" ->{ + transcriptionActions(call,result) + } + else -> { result.notImplemented() } @@ -500,6 +511,26 @@ class HmssdkFlutterPlugin : } } + private var isTranscriptionListenerAdded = false + private fun transcriptionActions( + call: MethodCall, + result: Result + ){ + when(call.method){ + "add_transcript_listener" -> { + isTranscriptionListenerAdded = true + result.success(null) + } + "remove_transcript_listener" -> { + isTranscriptionListenerAdded = false + result.success(null) + } + else -> hmssdk?.let { + HMSTranscriptionAction.transcriptionAction(call,result,it) + } + } + } + private fun pollActions( call: MethodCall, result: Result, @@ -531,6 +562,7 @@ class HmssdkFlutterPlugin : hlsPlayerChannel?.setStreamHandler(null) ?: Log.e("Channel Error", "HLS Player channel not found") pollsEventChannel?.setStreamHandler(null) ?: Log.e("Channel Error", "polls event channel not found") whiteboardEventChannel?.setStreamHandler(null) ?: Log.e("Channel Error", "whiteboard event channel not found") + transcriptionEventChannel?.setStreamHandler(null)?: Log.e("Channel Error", "transcription event channel not found") eventSink = null previewSink = null rtcSink = null @@ -539,6 +571,7 @@ class HmssdkFlutterPlugin : hlsPlayerSink = null pollsSink = null whiteboardSink = null + transcriptionSink = null hmssdkFlutterPlugin = null hmsBinaryMessenger = null hmsTextureRegistry = null @@ -657,22 +690,21 @@ class HmssdkFlutterPlugin : ) { val nameOfEventSink = (arguments as HashMap)["name"] - if (nameOfEventSink!! == "meeting") { - this.eventSink = events - } else if (nameOfEventSink == "preview") { - this.previewSink = events - } else if (nameOfEventSink == "logs") { - this.logsSink = events - } else if (nameOfEventSink == "rtc_stats") { - this.rtcSink = events - } else if (nameOfEventSink == "session_store") { - this.sessionStoreSink = events - } else if (nameOfEventSink == "hls_player") { - this.hlsPlayerSink = events - } else if (nameOfEventSink == "polls") { - this.pollsSink = events - } else if (nameOfEventSink == "whiteboard") { - this.whiteboardSink = events + nameOfEventSink?.let { eventSink -> + when(eventSink){ + "meeting" -> this.eventSink = events + "preview" -> this.previewSink = events + "logs" -> this.logsSink = events + "rtc_stats" -> this.rtcSink = events + "session_store" -> this.sessionStoreSink = events + "hls_player" -> this.hlsPlayerSink = events + "polls" -> this.pollsSink = events + "whiteboard" -> this.whiteboardSink = events + "transcription" -> this.transcriptionSink = events + else -> Log.e("Event Sink Error", "No sink with given name found") + } + }?:run{ + HMSErrorLogger.logError("onListen", "name of event sink is null", "NULL ERROR") } } @@ -1479,6 +1511,28 @@ class HmssdkFlutterPlugin : eventSink?.success(args) } } + + override fun onTranscripts(transcripts: HmsTranscripts) { + super.onTranscripts(transcripts) + + /** + * If transcription listener is added in the application + * then only we send data from here + */ + if(isTranscriptionListenerAdded){ + val args = HashMap() + args["event_name"] = "on_transcripts" + + val transcriptsList = ArrayList>() + transcripts.transcripts.forEach { _transcript -> + transcriptsList.add(HMSTranscriptExtension.toDictionary(_transcript)) + } + args["data"] = transcriptsList + CoroutineScope(Dispatchers.Main).launch { + transcriptionSink?.success(args) + } + } + } } private val hmsPreviewListener = diff --git a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/hms_role_components/PermissionParamsExtension.kt b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/hms_role_components/PermissionParamsExtension.kt index 5e7b9c8f7..c449bfe89 100644 --- a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/hms_role_components/PermissionParamsExtension.kt +++ b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/hms_role_components/PermissionParamsExtension.kt @@ -1,5 +1,7 @@ package live.hms.hmssdk_flutter.hms_role_components +import live.hms.hmssdk_flutter.HMSTranscriptExtension +import live.hms.video.sdk.models.role.HMSTranscriptionPermissions import live.hms.video.sdk.models.role.HMSWhiteBoardPermission import live.hms.video.sdk.models.role.PermissionsParams @@ -21,6 +23,7 @@ class PermissionParamsExtension { permissionsParams.whiteboard.let { args["whiteboard_permission"] = getMapFromHMSWhiteboardPermission(it) } + args["transcription_permission"] = getMapFromHMSTranscriptionPermissionsList(permissionsParams.transcriptions) return args } @@ -33,5 +36,18 @@ class PermissionParamsExtension { return permission } + + private fun getMapFromHMSTranscriptionPermissionsList(hmsTranscriptionPermission: List): ArrayList>{ + val transcriptionPermissionList = ArrayList>() + + hmsTranscriptionPermission.forEach {transcriptionPermission -> + val transcriptionPermissionMap = HashMap() + transcriptionPermissionMap["mode"] = HMSTranscriptExtension.getStringFromTranscriptionMode(transcriptionPermission.mode) + transcriptionPermissionMap["admin"] = transcriptionPermission.admin + + transcriptionPermissionList.add(transcriptionPermissionMap) + } + return transcriptionPermissionList + } } } diff --git a/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/methods/HMSTranscriptionAction.kt b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/methods/HMSTranscriptionAction.kt new file mode 100644 index 000000000..cf6f48a84 --- /dev/null +++ b/packages/hmssdk_flutter/android/src/main/kotlin/live/hms/hmssdk_flutter/methods/HMSTranscriptionAction.kt @@ -0,0 +1,60 @@ +package live.hms.hmssdk_flutter.methods + +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import live.hms.hmssdk_flutter.HMSCommonAction +import live.hms.hmssdk_flutter.HMSErrorLogger +import live.hms.hmssdk_flutter.HMSTranscriptExtension +import live.hms.video.sdk.HMSSDK + +class HMSTranscriptionAction { + + companion object{ + + fun transcriptionAction(call: MethodCall, + result: MethodChannel.Result, + hmssdk: HMSSDK?){ + when(call.method){ + "start_real_time_transcription" -> { + startRealTimeTranscription(call,result,hmssdk) + } + "stop_real_time_transcription" -> { + stopRealTimeTranscription(call,result,hmssdk) + } + } + } + + /** + * [startRealTimeTranscription] starts the transcription for everyone in the room + */ + private fun startRealTimeTranscription(call: MethodCall, result: MethodChannel.Result, hmssdk: HMSSDK?){ + + val mode = call.argument("mode") as? String + + mode?.let { _mode -> + val transcriptionMode = HMSTranscriptExtension.getTranscriptionModeFromString(_mode) + transcriptionMode?.let { + hmssdk?.startRealTimeTranscription(it, HMSCommonAction.getActionListener(result)) + }?:run{ + HMSErrorLogger.logError("startRealTimeTranscription","Mode is null","NULL Error") + } + } + } + + /** + * [stopRealTimeTranscription] starts the transcription for everyone in the room + */ + private fun stopRealTimeTranscription(call: MethodCall, result: MethodChannel.Result, hmssdk: HMSSDK?){ + val mode = call.argument("mode") as? String + + mode?.let { _mode -> + val transcriptionMode = HMSTranscriptExtension.getTranscriptionModeFromString(_mode) + transcriptionMode?.let { + hmssdk?.stopRealTimeTranscription(it, HMSCommonAction.getActionListener(result)) + }?:run{ + HMSErrorLogger.returnArgumentsError("mode is a required parameter cannot be null") + } + } + } + } +} \ No newline at end of file diff --git a/packages/hmssdk_flutter/example/ExampleAppChangelog.txt b/packages/hmssdk_flutter/example/ExampleAppChangelog.txt index 8414247b2..ab9d42beb 100644 --- a/packages/hmssdk_flutter/example/ExampleAppChangelog.txt +++ b/packages/hmssdk_flutter/example/ExampleAppChangelog.txt @@ -1,7 +1,10 @@ -Board: https://100ms.atlassian.net/jira/software/projects/FLUT/boards/34/ +Board: https://app.devrev.ai/100ms/vistas/vista-250 -- Move all the permissions from app to sdk -https://100ms.atlassian.net/browse/FLUT-253 +- Move all the permissions from app to SDK +https://app.devrev.ai/100ms/works/ISS-10106 + +- Live transcription in WebRTC mode +https://app.devrev.ai/100ms/works/ISS-22680 Room Kit: 1.1.4 Core SDK: 1.10.4 diff --git a/packages/hmssdk_flutter/example/android/app/build.gradle b/packages/hmssdk_flutter/example/android/app/build.gradle index 53e241146..acaeb6de6 100644 --- a/packages/hmssdk_flutter/example/android/app/build.gradle +++ b/packages/hmssdk_flutter/example/android/app/build.gradle @@ -36,8 +36,8 @@ android { applicationId "live.hms.flutter" minSdkVersion 21 targetSdkVersion 34 - versionCode 503 - versionName "1.5.203" + versionCode 507 + versionName "1.5.207" } signingConfigs { diff --git a/packages/hmssdk_flutter/example/ios/Gemfile b/packages/hmssdk_flutter/example/ios/Gemfile index a24614677..3e0c66695 100644 --- a/packages/hmssdk_flutter/example/ios/Gemfile +++ b/packages/hmssdk_flutter/example/ios/Gemfile @@ -1,6 +1,6 @@ source "https://rubygems.org" -gem "fastlane", "~> 2.220.0" +gem "fastlane", "~> 2.221.0" gem "activesupport", "= 7.0.8" plugins_path = File.join(File.dirname(__FILE__), 'fastlane', 'Pluginfile') diff --git a/packages/hmssdk_flutter/example/ios/Gemfile.lock b/packages/hmssdk_flutter/example/ios/Gemfile.lock index bac543212..09a15366e 100644 --- a/packages/hmssdk_flutter/example/ios/Gemfile.lock +++ b/packages/hmssdk_flutter/example/ios/Gemfile.lock @@ -15,17 +15,17 @@ GEM artifactory (3.0.17) atomos (0.1.3) aws-eventstream (1.3.0) - aws-partitions (1.929.0) - aws-sdk-core (3.196.1) + aws-partitions (1.944.0) + aws-sdk-core (3.197.0) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.81.0) - aws-sdk-core (~> 3, >= 3.193.0) + aws-sdk-kms (1.85.0) + aws-sdk-core (~> 3, >= 3.197.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.151.0) - aws-sdk-core (~> 3, >= 3.194.0) + aws-sdk-s3 (1.152.3) + aws-sdk-core (~> 3, >= 3.197.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) aws-sigv4 (1.8.0) @@ -74,7 +74,7 @@ GEM faraday_middleware (1.2.0) faraday (~> 1.0) fastimage (2.3.1) - fastlane (2.220.0) + fastlane (2.221.0) CFPropertyList (>= 2.3, < 4.0.0) addressable (>= 2.8, < 3.0.0) artifactory (~> 3.0) @@ -161,16 +161,16 @@ GEM os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) highline (2.0.3) - http-cookie (1.0.5) + http-cookie (1.0.6) domain_name (~> 0.5) httpclient (2.8.3) i18n (1.14.5) concurrent-ruby (~> 1.0) jmespath (1.6.2) json (2.7.2) - jwt (2.8.1) + jwt (2.8.2) base64 - mini_magick (4.12.0) + mini_magick (4.13.1) mini_mime (1.1.5) minitest (5.22.3) multi_json (1.15.0) @@ -181,14 +181,15 @@ GEM optparse (0.5.0) os (1.1.4) plist (3.7.1) - public_suffix (5.0.5) + public_suffix (5.1.1) rake (13.2.1) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.2.6) + rexml (3.2.9) + strscan rouge (2.0.7) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -201,6 +202,7 @@ GEM simctl (1.6.10) CFPropertyList naturally + strscan (3.1.0) terminal-notifier (2.0.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) @@ -231,7 +233,7 @@ PLATFORMS DEPENDENCIES activesupport (= 7.0.8) - fastlane (~> 2.220.0) + fastlane (~> 2.221.0) fastlane-plugin-firebase_app_distribution fastlane-plugin-versioning diff --git a/packages/hmssdk_flutter/example/ios/Podfile.lock b/packages/hmssdk_flutter/example/ios/Podfile.lock index e0ed2d933..1bdbf7f5f 100644 --- a/packages/hmssdk_flutter/example/ios/Podfile.lock +++ b/packages/hmssdk_flutter/example/ios/Podfile.lock @@ -133,7 +133,7 @@ PODS: - HMSSDK (1.12.0): - HMSAnalyticsSDK (= 0.0.2) - HMSWebRTC (= 1.0.6169) - - hmssdk_flutter (1.10.3): + - hmssdk_flutter (1.10.4): - Flutter - HMSBroadcastExtensionSDK (= 0.0.9) - HMSHLSPlayerSDK (= 0.0.2) @@ -316,7 +316,7 @@ SPEC CHECKSUMS: HMSHLSPlayerSDK: 6a54ad4d12f3dc2270d1ecd24019d71282a4f6a3 HMSNoiseCancellationModels: a3bda1405a16015632f4bcabd46ce48f35103b02 HMSSDK: 137107663eedc276c22639b2ec941c1f14f75d23 - hmssdk_flutter: cb46ccd6b59efc2f0b9ab9548b7addc95b3002ac + hmssdk_flutter: 869c38a41e8fae0bf78d47740beb7889b4b5837a HMSWebRTC: 8f51ba33a0e505e17ebf3d7b37bcdca266751a13 image_picker_ios: 99dfe1854b4fa34d0364e74a78448a0151025425 MLImage: 7bb7c4264164ade9bf64f679b40fb29c8f33ee9b diff --git a/packages/hmssdk_flutter/example/ios/Runner/Info.plist b/packages/hmssdk_flutter/example/ios/Runner/Info.plist index 15a6a21d9..954370388 100644 --- a/packages/hmssdk_flutter/example/ios/Runner/Info.plist +++ b/packages/hmssdk_flutter/example/ios/Runner/Info.plist @@ -21,7 +21,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.5.203 + 1.5.207 CFBundleSignature ???? CFBundleURLTypes @@ -48,7 +48,7 @@ CFBundleVersion - 503 + 507 ITSAppUsesNonExemptEncryption LSApplicationCategoryType diff --git a/packages/hmssdk_flutter/example/lib/qr_code_screen.dart b/packages/hmssdk_flutter/example/lib/qr_code_screen.dart index 6d4c04bba..165bc8338 100644 --- a/packages/hmssdk_flutter/example/lib/qr_code_screen.dart +++ b/packages/hmssdk_flutter/example/lib/qr_code_screen.dart @@ -37,63 +37,64 @@ class _QRCodeScreenState extends State { if (barcodes.isNotEmpty) { log(barcodes[0].rawValue ?? ""); String? rawValue = barcodes[0].rawValue; - if (rawValue != null) { - FocusManager.instance.primaryFocus?.unfocus(); - - Map? endPoints; - if (rawValue.trim().contains("app.100ms.live")) { - List? roomData = RoomService.getCode(rawValue.trim()); - - //If the link is not valid then we might not get the code and whether the link is a - //PROD or QA so we return the error in this case - if (roomData == null || roomData.isEmpty) { - return; - } - - ///************************************************************************************************** */ - - ///This section can be safely commented out as it's only required for 100ms internal usage - - //qaTokenEndPoint is only required for 100ms internal testing - //It can be removed and should not affect the join method call - //For _endPoint just pass it as null - //the endPoint parameter in getAuthTokenByRoomCode can be passed as null - //Pass the layoutAPIEndPoint as null the qa endPoint is only for 100ms internal testing - - ///If you wish to set your own token end point then you can pass it in the endPoints map - ///The key for the token end point is "tokenEndPointKey" - ///The key for the init end point is "initEndPointKey" - ///The key for the layout api end point is "layoutAPIEndPointKey" - if (roomData[1] == "false") { - endPoints = RoomService.setEndPoints(); - } - - ///************************************************************************************************** */ - - Constant.roomCode = roomData[0] ?? ''; - } else { - Constant.roomCode = rawValue.trim(); + FocusManager.instance.primaryFocus?.unfocus(); + + if (rawValue == null) { + return; + } + Map? endPoints; + if (rawValue.trim().contains("app.100ms.live")) { + List? roomData = RoomService.getCode(rawValue.trim()); + + //If the link is not valid then we might not get the code and whether the link is a + //PROD or QA so we return the error in this case + if (roomData == null || roomData.isEmpty) { + return; } - Utilities.saveStringData(key: "meetingLink", value: rawValue.trim()); - await initForegroundTask(); - Navigator.of(context).pushReplacement(MaterialPageRoute( - builder: (_) => WithForegroundTask( - child: HMSPrebuilt( - roomCode: Constant.roomCode, - onLeave: stopForegroundTask, - options: HMSPrebuiltOptions( - userName: AppDebugConfig.nameChangeOnPreview - ? null - : "Flutter User", - userId: widget.uuidString, - endPoints: endPoints, - iOSScreenshareConfig: HMSIOSScreenshareConfig( - appGroup: "group.flutterhms", - preferredExtension: - "live.100ms.flutter.FlutterBroadcastUploadExtension"), - enableNoiseCancellation: true)), - ))); + + ///************************************************************************************************** */ + + ///This section can be safely commented out as it's only required for 100ms internal usage + + //qaTokenEndPoint is only required for 100ms internal testing + //It can be removed and should not affect the join method call + //For _endPoint just pass it as null + //the endPoint parameter in getAuthTokenByRoomCode can be passed as null + //Pass the layoutAPIEndPoint as null the qa endPoint is only for 100ms internal testing + + ///If you wish to set your own token end point then you can pass it in the endPoints map + ///The key for the token end point is "tokenEndPointKey" + ///The key for the init end point is "initEndPointKey" + ///The key for the layout api end point is "layoutAPIEndPointKey" + if (roomData[1] == "false") { + endPoints = RoomService.setEndPoints(); + } + + ///************************************************************************************************** */ + + Constant.roomCode = roomData[0] ?? ''; + } else { + Constant.roomCode = rawValue.trim(); } + Utilities.saveStringData(key: "meetingLink", value: rawValue.trim()); + await initForegroundTask(); + Navigator.of(context).pushReplacement(MaterialPageRoute( + builder: (_) => WithForegroundTask( + child: HMSPrebuilt( + roomCode: Constant.roomCode, + onLeave: stopForegroundTask, + options: HMSPrebuiltOptions( + userName: AppDebugConfig.nameChangeOnPreview + ? null + : "Flutter User", + userId: widget.uuidString, + endPoints: endPoints, + iOSScreenshareConfig: HMSIOSScreenshareConfig( + appGroup: "group.flutterhms", + preferredExtension: + "live.100ms.flutter.FlutterBroadcastUploadExtension"), + enableNoiseCancellation: true)), + ))); } } catch (e) { log(e.toString()); diff --git a/packages/hmssdk_flutter/ios/Classes/Actions/HMSTranscriptionAction.swift b/packages/hmssdk_flutter/ios/Classes/Actions/HMSTranscriptionAction.swift new file mode 100644 index 000000000..004ca0361 --- /dev/null +++ b/packages/hmssdk_flutter/ios/Classes/Actions/HMSTranscriptionAction.swift @@ -0,0 +1,47 @@ +// +// HMSTranscriptionAction.swift +// hmssdk_flutter +// +// Created by Pushpam on 14/06/24. +// + +import Foundation +import HMSSDK + +class HMSTranscriptionAction{ + + static func transcriptionActions(_ call: FlutterMethodCall, _ result: @escaping FlutterResult, _ hmssdk: HMSSDK?){ + switch call.method{ + case "start_real_time_transcription": + startRealTimeTranscription(result, hmssdk) + case "stop_real_time_transcription": + stopRealTimeTranscription(result, hmssdk) + default: + result(FlutterMethodNotImplemented) + } + } + + private static func startRealTimeTranscription(_ result: @escaping FlutterResult, _ hmssdk: HMSSDK?){ + + hmssdk?.startTranscription(){ _, error in + if let error = error{ + result(HMSErrorExtension.toDictionary(error)) + }else{ + result(nil) + } + } + } + + + private static func stopRealTimeTranscription(_ result: @escaping FlutterResult, _ hmssdk: HMSSDK?){ + hmssdk?.stopTranscription(){_, error in + if let error = error{ + result(HMSErrorExtension.toDictionary(error)) + }else{ + result(nil) + } + } + } + + +} diff --git a/packages/hmssdk_flutter/ios/Classes/Models/HMSPermissionExtension.swift b/packages/hmssdk_flutter/ios/Classes/Models/HMSPermissionExtension.swift index 5c28102b9..009c0a2fe 100644 --- a/packages/hmssdk_flutter/ios/Classes/Models/HMSPermissionExtension.swift +++ b/packages/hmssdk_flutter/ios/Classes/Models/HMSPermissionExtension.swift @@ -22,11 +22,12 @@ class HMSPermissionExtension { "un_mute": permission.unmute ?? false, "poll_read": permission.pollRead ?? false, "poll_write": permission.pollWrite ?? false, - "whiteboard_permission": getMapFromHMSWhiteboardPermission(hmsWhiteboardPermission: permission.whiteboard) + "whiteboard_permission": getMapFromHMSWhiteboardPermission(hmsWhiteboardPermission: permission.whiteboard), + "transcription_permission": getMapFromHMSTranscriptionPermissionList(hmsTranscriptionPermissions: permission.transcriptions) ] } - static func getMapFromHMSWhiteboardPermission(hmsWhiteboardPermission: HMSWhiteboardPermissions?) -> [String: Any?]? { + private static func getMapFromHMSWhiteboardPermission(hmsWhiteboardPermission: HMSWhiteboardPermissions?) -> [String: Any?]? { guard let hmsWhiteboardPermission = hmsWhiteboardPermission else { @@ -41,5 +42,27 @@ class HMSPermissionExtension { return permission } + + private static func getMapFromHMSTranscriptionPermissionList(hmsTranscriptionPermissions: [HMSTranscriptionPermissions]?) -> [[String:Any?]]?{ + + + guard let hmsTranscriptionPermissions = hmsTranscriptionPermissions + else{ + return nil + } + + var transcriptionPermissions = [[String:Any?]]() + + hmsTranscriptionPermissions.forEach{ + var permission = [String:Any?]() + + permission["mode"] = $0.mode + permission["admin"] = $0.admin + transcriptionPermissions.append(permission) + } + + return transcriptionPermissions + } + } diff --git a/packages/hmssdk_flutter/ios/Classes/Models/HMSRoomExtension.swift b/packages/hmssdk_flutter/ios/Classes/Models/HMSRoomExtension.swift index c99622137..0e0a83b41 100644 --- a/packages/hmssdk_flutter/ios/Classes/Models/HMSRoomExtension.swift +++ b/packages/hmssdk_flutter/ios/Classes/Models/HMSRoomExtension.swift @@ -49,6 +49,8 @@ class HMSRoomExtension { dict["hls_recording_state"] = HMSStreamingStateExtension.toDictionary(hlsRecording: room.hlsRecordingState) + dict["transcriptions"] = HMSTranscriptExtension.getMapFromTranscriptionsStateList(transcriptionStates: room.transcriptionStates) + return dict } @@ -68,6 +70,8 @@ class HMSRoomExtension { return "hls_recording_state_updated" case .peerCountUpdated: return "room_peer_count_updated" + case .transcriptionStateUpdated: + return "transcriptions_updated" default: return "unknown_update" } diff --git a/packages/hmssdk_flutter/ios/Classes/Models/HMSTranscriptExtension.swift b/packages/hmssdk_flutter/ios/Classes/Models/HMSTranscriptExtension.swift new file mode 100644 index 000000000..2758ae4b6 --- /dev/null +++ b/packages/hmssdk_flutter/ios/Classes/Models/HMSTranscriptExtension.swift @@ -0,0 +1,82 @@ +// +// HMSTranscriptExtension.swift +// hmssdk_flutter +// +// Created by Pushpam on 14/06/24. +// + +import Foundation +import HMSSDK + +class HMSTranscriptExtension{ + + static func toDictionary(hmsTranscript: HMSTranscript) -> [String:Any?]{ + + var args = [String:Any?]() + + args["start"] = hmsTranscript.start + args["end"] = hmsTranscript.end + args["transcript"] = hmsTranscript.transcript + args["peer_id"] = hmsTranscript.peer.peerID + args["peer_name"] = hmsTranscript.peer.name + args["is_final"] = hmsTranscript.isFinal + + return args + } + + static func getMapFromTranscriptionsStateList(transcriptionStates: [HMSTranscriptionState]?) -> [[String:Any?]]?{ + + if let transcripts = transcriptionStates{ + var transcriptStates = [[String:Any?]]() + + transcripts.forEach{ + transcriptStates.append(getMapFromTranscriptionState(transcriptionState: $0)) + } + return transcriptStates + }else{ + return nil + } + + } + + private static func getMapFromTranscriptionState(transcriptionState: HMSTranscriptionState) -> [String:Any?]{ + + var args = [String:Any?]() + + if let startedAt = transcriptionState.startedAt{ + args["started_at"] = Int(startedAt.timeIntervalSince1970 * 1000) + } + + if let stoppedAt = transcriptionState.stoppedAt{ + args["stopped_at"] = Int(stoppedAt.timeIntervalSince1970 * 1000) + } + + if let updatedAt = transcriptionState.updatedAt{ + args["updated_at"] = Int(updatedAt.timeIntervalSince1970 * 1000) + } + + args["state"] = getStringFromTranscriptionState(transcriptionState: transcriptionState.state) + + args["mode"] = transcriptionState.mode + + return args + } + + private static func getStringFromTranscriptionState(transcriptionState: HMSTranscriptionStatus) -> String?{ + switch transcriptionState{ + case .failed: + return "failed" + case .started: + return "started" + case .stopped: + return "stopped" + case .starting: + return "initialized" + default: + return nil + } + } + + + +} diff --git a/packages/hmssdk_flutter/ios/Classes/SwiftHmssdkFlutterPlugin.swift b/packages/hmssdk_flutter/ios/Classes/SwiftHmssdkFlutterPlugin.swift index fd98f14f5..640681cde 100644 --- a/packages/hmssdk_flutter/ios/Classes/SwiftHmssdkFlutterPlugin.swift +++ b/packages/hmssdk_flutter/ios/Classes/SwiftHmssdkFlutterPlugin.swift @@ -18,7 +18,8 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene var hlsPlayerChannel: FlutterEventChannel? var pollsEventChannel: FlutterEventChannel? var whiteboardEventChannel: FlutterEventChannel? - + var transcriptionEventChannel: FlutterEventChannel? + var eventSink: FlutterEventSink? var previewSink: FlutterEventSink? var logsSink: FlutterEventSink? @@ -27,6 +28,7 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene var hlsPlayerSink: FlutterEventSink? var pollsEventSink: FlutterEventSink? var whiteboardEventSink: FlutterEventSink? + var transcriptionSink: FlutterEventSink? var roleChangeRequest: HMSRoleChangeRequest? @@ -61,7 +63,8 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene let hlsChannel = FlutterEventChannel(name: "hls_player_channel", binaryMessenger: registrar.messenger()) let pollsChannel = FlutterEventChannel(name: "polls_event_channel", binaryMessenger: registrar.messenger()) let whiteboardChannel = FlutterEventChannel(name: "whiteboard_event_channel", binaryMessenger: registrar.messenger()) - + let transcriptionChannel = FlutterEventChannel(name: "transcription_event_channel", binaryMessenger: registrar.messenger()) + let instance = SwiftHmssdkFlutterPlugin(channel: channel, meetingEventChannel: eventChannel, previewEventChannel: previewChannel, @@ -70,7 +73,8 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene sessionEventChannel: sessionChannel, hlsPlayerChannel: hlsChannel, pollsEventChannel: pollsChannel, - whiteboardEventChannel: whiteboardChannel) + whiteboardEventChannel: whiteboardChannel, + transcriptionEventChannel: transcriptionChannel) let videoViewFactory = HMSFlutterPlatformViewFactory(plugin: instance) registrar.register(videoViewFactory, withId: "HMSFlutterPlatformView") @@ -86,6 +90,7 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene hlsChannel.setStreamHandler(instance) pollsChannel.setStreamHandler(instance) whiteboardChannel.setStreamHandler(instance) + transcriptionChannel.setStreamHandler(instance) registrar.addMethodCallDelegate(instance, channel: channel) } @@ -98,7 +103,8 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene sessionEventChannel: FlutterEventChannel, hlsPlayerChannel: FlutterEventChannel, pollsEventChannel: FlutterEventChannel, - whiteboardEventChannel: FlutterEventChannel) { + whiteboardEventChannel: FlutterEventChannel, + transcriptionEventChannel: FlutterEventChannel) { self.channel = channel self.meetingEventChannel = meetingEventChannel @@ -109,6 +115,7 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene self.hlsPlayerChannel = hlsPlayerChannel self.pollsEventChannel = pollsEventChannel self.whiteboardEventChannel = whiteboardEventChannel + self.transcriptionEventChannel = transcriptionEventChannel } public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { @@ -135,6 +142,8 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene pollsEventSink = events case "whiteboard": whiteboardEventSink = events + case "transcription": + transcriptionSink = events default: return FlutterError(code: #function, message: "invalid event sink name", details: arguments) } @@ -195,6 +204,12 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene } else { print(#function, "whiteboardEventChannel not found") } + if transcriptionEventChannel != nil { + transcriptionEventChannel!.setStreamHandler(nil) + transcriptionSink = nil + } else { + print(#function, "transcriptionEventChannel not found") + } } public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { @@ -345,6 +360,9 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene case "enable_virtual_background", "disable_virtual_background", "enable_blur_background", "disable_blur_background", "change_virtual_background", "is_virtual_background_supported": vbAction.performActions(call, result) + case "start_real_time_transcription", "stop_real_time_transcription","add_transcript_listener", "remove_transcript_listener": + transcriptionActions(call, result) + default: result(FlutterMethodNotImplemented) } @@ -525,6 +543,20 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene HMSWhiteboardAction.whiteboardActions(call, result, hmsSDK) } } + + private var isTranscriptionListenerAdded = false + private func transcriptionActions(_ call: FlutterMethodCall, _ result: @escaping FlutterResult){ + switch call.method{ + case "add_transcript_listener": + isTranscriptionListenerAdded = true + result(nil) + case "remove_transcript_listener": + isTranscriptionListenerAdded = false + result(nil) + default: + HMSTranscriptionAction.transcriptionActions(call, result, hmsSDK) + } + } // MARK: - Track Setting var audioMixerSourceMap = [String: HMSAudioNode]() @@ -1565,6 +1597,28 @@ public class SwiftHmssdkFlutterPlugin: NSObject, FlutterPlugin, HMSUpdateListene previewSink?(data) } } + + public func on(transcripts: HMSTranscripts){ + + /** + * If transcription listener is added in the application + * then only we send data from here + */ + if(isTranscriptionListenerAdded){ + var transcriptList = [[String:Any?]]() + + transcripts.transcripts.forEach{ + transcriptList.append(HMSTranscriptExtension.toDictionary(hmsTranscript: $0)) + } + + var data = [String:Any?]() + + data["event_name"] = "on_transcripts" + data["data"] = transcriptList + self.transcriptionSink?(data) + } + } + // MARK: - RTC Stats Listeners diff --git a/packages/hmssdk_flutter/lib/hmssdk_flutter.dart b/packages/hmssdk_flutter/lib/hmssdk_flutter.dart index 67780008a..d7ff52911 100644 --- a/packages/hmssdk_flutter/lib/hmssdk_flutter.dart +++ b/packages/hmssdk_flutter/lib/hmssdk_flutter.dart @@ -27,6 +27,8 @@ export 'src/enum/hms_peer_type.dart'; export 'src/enum/hms_hls_playlist_type.dart'; export 'src/enum/hms_whiteboard_listener_method.dart'; export 'src/enum/hms_whiteboard_state.dart'; +export 'src/enum/hms_transcription_mode.dart'; +export 'src/enum/hms_transcription_state.dart'; //EXCEPTIONS export 'src/exceptions/hms_exception.dart'; @@ -123,6 +125,10 @@ export 'src/model/whiteboard/hms_whiteboard_permission.dart'; export 'src/model/whiteboard/hms_whiteboard_update_listener.dart'; export 'src/model/hls_player/hms_hls_layer.dart'; export 'src/model/hms_video_filter.dart'; +export 'src/model/transcription/hms_transcription_permission.dart'; +export 'src/model/transcription//hms_transcript_listener.dart'; +export 'src/model/transcription/hms_transcription.dart'; +export 'src/model/transcription/hms_transcription_controller.dart'; //Views export 'src/ui/meeting/hms_texture_view.dart'; diff --git a/packages/hmssdk_flutter/lib/src/common/platform_methods.dart b/packages/hmssdk_flutter/lib/src/common/platform_methods.dart index d49355bc6..7cb9c51bf 100644 --- a/packages/hmssdk_flutter/lib/src/common/platform_methods.dart +++ b/packages/hmssdk_flutter/lib/src/common/platform_methods.dart @@ -246,7 +246,13 @@ enum PlatformMethod { changeVirtualBackground, enableBlurBackground, disableBlurBackground, - isVirtualBackgroundSupported + isVirtualBackgroundSupported, + + ///Transcription methods + startRealTimeTranscription, + stopRealTimeTranscription, + addTranscriptListener, + removeTranscriptListener } extension PlatformMethodValues on PlatformMethod { @@ -626,6 +632,15 @@ extension PlatformMethodValues on PlatformMethod { case PlatformMethod.isVirtualBackgroundSupported: return "is_virtual_background_supported"; + case PlatformMethod.startRealTimeTranscription: + return "start_real_time_transcription"; + case PlatformMethod.stopRealTimeTranscription: + return "stop_real_time_transcription"; + case PlatformMethod.addTranscriptListener: + return "add_transcript_listener"; + case PlatformMethod.removeTranscriptListener: + return "remove_transcript_listener"; + default: return 'unknown'; } @@ -1005,6 +1020,15 @@ extension PlatformMethodValues on PlatformMethod { case "is_virtual_background_supported": return PlatformMethod.isVirtualBackgroundSupported; + case "start_real_time_transcription": + return PlatformMethod.startRealTimeTranscription; + case "stop_real_time_transcription": + return PlatformMethod.stopRealTimeTranscription; + case "add_transcript_listener": + return PlatformMethod.addTranscriptListener; + case "remove_transcript_listener": + return PlatformMethod.removeTranscriptListener; + default: return PlatformMethod.unknown; } diff --git a/packages/hmssdk_flutter/lib/src/enum/hms_room_update.dart b/packages/hmssdk_flutter/lib/src/enum/hms_room_update.dart index 57e205245..c013a8801 100644 --- a/packages/hmssdk_flutter/lib/src/enum/hms_room_update.dart +++ b/packages/hmssdk_flutter/lib/src/enum/hms_room_update.dart @@ -23,6 +23,9 @@ enum HMSRoomUpdate { //When peer Count is changed roomPeerCountUpdated, + ///When transcription state is updated + transcriptionsUpdated, + ///Default Update defaultUpdate } @@ -54,6 +57,9 @@ extension HMSRoomUpdateValues on HMSRoomUpdate { case 'room_peer_count_updated': return HMSRoomUpdate.roomPeerCountUpdated; + case 'transcriptions_updated': + return HMSRoomUpdate.transcriptionsUpdated; + default: return HMSRoomUpdate.defaultUpdate; } @@ -85,6 +91,9 @@ extension HMSRoomUpdateValues on HMSRoomUpdate { case HMSRoomUpdate.roomPeerCountUpdated: return 'room_peer_count_updated'; + case HMSRoomUpdate.transcriptionsUpdated: + return 'transcriptions_updated'; + default: return 'defaultUpdate'; } diff --git a/packages/hmssdk_flutter/lib/src/enum/hms_transcription_listener_method.dart b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_listener_method.dart new file mode 100644 index 000000000..aab86aade --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_listener_method.dart @@ -0,0 +1,15 @@ +///[HMSTranscriptionListenerMethod] contains method for `HMSTranscriptionListener` +enum HMSTranscriptionListenerMethod { onTranscripts, unknown } + +extension HMSTranscriptionListenerMethodValues + on HMSTranscriptionListenerMethod { + static HMSTranscriptionListenerMethod + getHMSTranscriptionListenerMethodFromString(String transcription) { + switch (transcription) { + case "on_transcripts": + return HMSTranscriptionListenerMethod.onTranscripts; + default: + return HMSTranscriptionListenerMethod.unknown; + } + } +} diff --git a/packages/hmssdk_flutter/lib/src/enum/hms_transcription_mode.dart b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_mode.dart new file mode 100644 index 000000000..0f670741b --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_mode.dart @@ -0,0 +1,16 @@ +///[HMSTranscriptionMode] is an enum class that defines the transcription mode of the meeting. +enum HMSTranscriptionMode { caption, live, unknown } + +extension HMSTranscriptionModeValues on HMSTranscriptionMode { + static HMSTranscriptionMode getHMSTranscriptionModeFromString( + String transcription) { + switch (transcription) { + case "caption": + return HMSTranscriptionMode.caption; + case "live": + return HMSTranscriptionMode.live; + default: + return HMSTranscriptionMode.unknown; + } + } +} diff --git a/packages/hmssdk_flutter/lib/src/enum/hms_transcription_state.dart b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_state.dart new file mode 100644 index 000000000..3fe4561bd --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/enum/hms_transcription_state.dart @@ -0,0 +1,20 @@ +///[HMSTranscriptionState] is an enum class which defines the state of the transcription +enum HMSTranscriptionState { started, stopped, initialized, failed, unknown } + +extension HMSTranscriptionStateValues on HMSTranscriptionState { + static HMSTranscriptionState getHMSTranscriptionStateFromString( + String transcription) { + switch (transcription) { + case "started": + return HMSTranscriptionState.started; + case "stopped": + return HMSTranscriptionState.stopped; + case "initialized": + return HMSTranscriptionState.initialized; + case "failed": + return HMSTranscriptionState.failed; + default: + return HMSTranscriptionState.unknown; + } + } +} diff --git a/packages/hmssdk_flutter/lib/src/model/hms_permissions.dart b/packages/hmssdk_flutter/lib/src/model/hms_permissions.dart index 312e2a019..273a4d44b 100644 --- a/packages/hmssdk_flutter/lib/src/model/hms_permissions.dart +++ b/packages/hmssdk_flutter/lib/src/model/hms_permissions.dart @@ -1,3 +1,4 @@ +import 'package:hmssdk_flutter/src/model/transcription/hms_transcription_permission.dart'; import 'package:hmssdk_flutter/src/model/whiteboard/hms_whiteboard_permission.dart'; ///100ms HMSPermissions @@ -15,6 +16,7 @@ class HMSPermissions { final bool? pollRead; final bool? pollWrite; final HMSWhiteboardPermission? whiteboard; + final List? transcription; HMSPermissions( {this.endRoom, @@ -27,7 +29,8 @@ class HMSPermissions { this.changeRole, this.pollRead, this.pollWrite, - this.whiteboard}); + this.whiteboard, + this.transcription}); factory HMSPermissions.fromMap(Map map) { return HMSPermissions( @@ -43,6 +46,12 @@ class HMSPermissions { pollWrite: map['poll_write'], whiteboard: map['whiteboard_permission'] != null ? HMSWhiteboardPermission.fromMap(map['whiteboard_permission']) + : null, + transcription: map['transcription_permission'] != null + ? map['transcription_permission'] + .map( + (e) => HMSTranscriptionPermission.fromMap(e)) + .toList() : null); } } diff --git a/packages/hmssdk_flutter/lib/src/model/hms_room.dart b/packages/hmssdk_flutter/lib/src/model/hms_room.dart index 3671372d0..1894db43a 100644 --- a/packages/hmssdk_flutter/lib/src/model/hms_room.dart +++ b/packages/hmssdk_flutter/lib/src/model/hms_room.dart @@ -3,6 +3,7 @@ import 'package:hmssdk_flutter/hmssdk_flutter.dart'; import 'package:hmssdk_flutter/src/model/hms_browser_recording_state.dart'; import 'package:hmssdk_flutter/src/model/hms_hls_recording_state.dart'; import 'package:hmssdk_flutter/src/model/hms_server_recording_state.dart'; +import 'package:hmssdk_flutter/src/model/transcription.dart'; import 'hms_hls_streaming_state.dart'; import 'hms_rtmp_streaming_state.dart'; @@ -22,6 +23,7 @@ class HMSRoom { String? name; String? metaData; bool isLarge; + List? transcriptions; HMSBrowserRecordingState? hmsBrowserRecordingState; HMSRtmpStreamingState? hmsRtmpStreamingState; HMSServerRecordingState? hmsServerRecordingState; @@ -39,6 +41,7 @@ class HMSRoom { this.name, required this.peers, required this.isLarge, + this.transcriptions, this.metaData, this.hmsServerRecordingState, this.hmsRtmpStreamingState, @@ -62,6 +65,17 @@ class HMSRoom { } } + List? transcriptions; + + if (map["transcriptions"] != null) { + map["transcriptions"].forEach((element) { + if (transcriptions == null) { + transcriptions = []; + } + transcriptions?.add(Transcription.fromMap(element)); + }); + } + return HMSRoom( hmsBrowserRecordingState: map["browser_recording_state"] != null ? HMSBrowserRecordingState.fromMap(map["browser_recording_state"]) @@ -85,7 +99,8 @@ class HMSRoom { metaData: map['meta_data'], peerCount: map["peer_count"] != null ? map["peer_count"] : 0, startedAt: map["started_at"] != null ? map["started_at"] : 0, - sessionId: map["session_id"] != null ? map["session_id"] : ""); + sessionId: map["session_id"] != null ? map["session_id"] : "", + transcriptions: transcriptions); } @override diff --git a/packages/hmssdk_flutter/lib/src/model/platform_method_response.dart b/packages/hmssdk_flutter/lib/src/model/platform_method_response.dart index 3d2cc531b..0d94ab62d 100644 --- a/packages/hmssdk_flutter/lib/src/model/platform_method_response.dart +++ b/packages/hmssdk_flutter/lib/src/model/platform_method_response.dart @@ -3,6 +3,7 @@ import 'package:hmssdk_flutter/hmssdk_flutter.dart'; import 'package:hmssdk_flutter/src/enum/hms_hls_playback_event_method.dart'; import 'package:hmssdk_flutter/src/enum/hms_key_change_listener_method.dart'; import 'package:hmssdk_flutter/src/enum/hms_logs_update_listener.dart'; +import 'package:hmssdk_flutter/src/enum/hms_transcription_listener_method.dart'; ///PlatformMethodResponse contains all the responses sent back from the platform /// @@ -103,3 +104,11 @@ class HMSWhiteboardListenerMethodResponse { HMSWhiteboardListenerMethodResponse( {required this.method, required this.data}); } + +class HMSTranscriptionListenerMethodResponse { + final HMSTranscriptionListenerMethod method; + final List data; + + HMSTranscriptionListenerMethodResponse( + {required this.method, required this.data}); +} diff --git a/packages/hmssdk_flutter/lib/src/model/transcription.dart b/packages/hmssdk_flutter/lib/src/model/transcription.dart new file mode 100644 index 000000000..5892251ad --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/model/transcription.dart @@ -0,0 +1,72 @@ +//Project imports +import 'package:hmssdk_flutter/src/enum/hms_transcription_mode.dart'; +import 'package:hmssdk_flutter/src/enum/hms_transcription_state.dart'; +import 'package:hmssdk_flutter/src/model/hms_date_extension.dart'; + +///[Transcription] is a class which includes the properties of a transcription +class Transcription { + ///[error] is the error object which contains the error code and message if any error occurs. + final TranscriptionError? error; + + ///[startedAt] is the time when the transcription started. + final DateTime? startedAt; + + ///[stoppedAt] is the time when the transcription stopped. + final DateTime? stoppedAt; + + ///[updatedAt] is the time when the transcription was last updated. + final DateTime? updatedAt; + + ///[state] is an enum of type [HMSTranscriptionState] which tells the state of the transcription. + final HMSTranscriptionState? state; + + ///[mode] is an enum of type [HMSTranscriptionMode] which tells the mode of the transcription. + final HMSTranscriptionMode? mode; + + Transcription( + {this.error, + this.startedAt, + this.stoppedAt, + this.updatedAt, + this.state, + this.mode}); + + factory Transcription.fromMap(Map map) { + return Transcription( + error: map["error"] != null + ? TranscriptionError.fromMap(map["error"]) + : null, + startedAt: map["started_at"] != null + ? HMSDateExtension.convertDateFromEpoch(map['started_at']) + : null, + stoppedAt: map["stopped_at"] != null + ? HMSDateExtension.convertDateFromEpoch(map['stopped_at']) + : null, + updatedAt: map["updated_at"] != null + ? HMSDateExtension.convertDateFromEpoch(map['updated_at']) + : null, + state: map["state"] != null + ? HMSTranscriptionStateValues.getHMSTranscriptionStateFromString( + map["state"]) + : null, + mode: map["mode"] != null + ? HMSTranscriptionModeValues.getHMSTranscriptionModeFromString( + map["mode"]) + : null); + } +} + +///[TranscriptionError] is a class which includes the properties of an error in transcription +class TranscriptionError { + ///[code] is the error code. + final int? code; + + ///[message] is the error message. + final String? message; + + TranscriptionError({required this.code, required this.message}); + + factory TranscriptionError.fromMap(Map map) { + return TranscriptionError(code: map['code'], message: map['message']); + } +} diff --git a/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcript_listener.dart b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcript_listener.dart new file mode 100644 index 000000000..01dd4ebe4 --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcript_listener.dart @@ -0,0 +1,10 @@ +//Project imports +import 'package:hmssdk_flutter/src/model/transcription/hms_transcription.dart'; + +///[HMSTranscriptListener] is the listener interface which listens to the transcription of the meeting. +/// +///Implement this listener in your class to get the transcription of the meeting. +abstract class HMSTranscriptListener { + ///[onTranscripts] is called when the transcription is received. + void onTranscripts({required List transcriptions}) {} +} diff --git a/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription.dart b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription.dart new file mode 100644 index 000000000..df7d9b61b --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription.dart @@ -0,0 +1,35 @@ +///[HMSTranscription] is a class which is used to represent a transcription. +class HMSTranscription { + final int start; + final int end; + + ///[transcript] is the text of the transcription. + final String transcript; + + ///[peerId] is the id of the speaker. + final String peerId; + + ///[peerName] is the name of the speaker. + final String? peerName; + + ///[isFinal] is a boolean which tells if the transcription is final or not. + final bool isFinal; + + HMSTranscription( + {required this.start, + required this.end, + required this.transcript, + required this.peerId, + required this.peerName, + required this.isFinal}); + + factory HMSTranscription.fromMap(Map map) { + return HMSTranscription( + start: map['start'], + end: map['end'], + transcript: map['transcript'], + peerId: map['peer_id'], + peerName: map['peer_name'], + isFinal: map['is_final']); + } +} diff --git a/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_controller.dart b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_controller.dart new file mode 100644 index 000000000..4e3ef45c1 --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_controller.dart @@ -0,0 +1,63 @@ +//Project imports +import 'package:hmssdk_flutter/hmssdk_flutter.dart'; +import 'package:hmssdk_flutter/src/service/platform_service.dart'; + +///[HMSTranscriptionController] is used to control transcription in the meeting. +abstract class HMSTranscriptionController { + ///[addListener] is used to add a listener to get the transcription of the meeting. + /// **parameters**: + /// + /// **listener** - [HMSTranscriptListener] instance to be attached + /// Learn more about [addListener] [here](https://www.100ms.live/docs/flutter/v2/how-to-guides/extend-capabilities/live-captions#step-1-add-hmstranscriptlistener-to-the-class-to-start-getting-transcriptions) + static void addListener({required HMSTranscriptListener listener}) { + PlatformService.addTranscriptListener(listener); + PlatformService.invokeMethod(PlatformMethod.addTranscriptListener); + } + + ///[removeListener] is used to remove the listener that was previously added. + ///Learn more about [removeListener] [here](https://www.100ms.live/docs/flutter/v2/how-to-guides/extend-capabilities/live-captions#step-3-to-stop-getting-transcriptions-remove-hmstranscriptlistener) + static void removeListener() { + PlatformService.removeTranscriptListener(); + PlatformService.invokeMethod(PlatformMethod.removeTranscriptListener); + } + + ///[startTranscription] is used to start the transcription of the meeting. + /// + /// **parameters**: + /// + /// **mode** - [HMSTranscriptionMode] to start the transcription in the meeting. Default is [HMSTranscriptionMode.caption] + /// + /// Refer [startTranscription](https://www.100ms.live/docs/flutter/v2/how-to-guides/extend-capabilities/live-captions#start-transcription) + static Future startTranscription( + {HMSTranscriptionMode mode = HMSTranscriptionMode.caption}) async { + var result = await PlatformService.invokeMethod( + PlatformMethod.startRealTimeTranscription, + arguments: {'mode': mode.name}); + + if (result != null) { + return HMSException.fromMap(result["error"]); + } else { + return null; + } + } + + ///[stopTranscription] is used to stop the transcription of the meeting. + /// + /// **parameters**: + /// + /// **mode** - [HMSTranscriptionMode] to stop the transcription in the meeting. Default is [HMSTranscriptionMode.caption] + /// + /// Refer [stopTranscription](https://www.100ms.live/docs/flutter/v2/how-to-guides/extend-capabilities/live-captions#stop-transcription) + static Future stopTranscription( + {HMSTranscriptionMode mode = HMSTranscriptionMode.caption}) async { + var result = await PlatformService.invokeMethod( + PlatformMethod.stopRealTimeTranscription, + arguments: {'mode': mode.name}); + + if (result != null) { + return HMSException.fromMap(result["error"]); + } else { + return null; + } + } +} diff --git a/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_permission.dart b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_permission.dart new file mode 100644 index 000000000..0d0af794c --- /dev/null +++ b/packages/hmssdk_flutter/lib/src/model/transcription/hms_transcription_permission.dart @@ -0,0 +1,21 @@ +//Project imports +import 'package:hmssdk_flutter/src/enum/hms_transcription_mode.dart'; + +///[HMSTranscriptionPermission] contains the permission of the user for transcription. +class HMSTranscriptionPermission { + ///[mode] is the transcription mode of the meeting. + final HMSTranscriptionMode mode; + + ///[admin] is a boolean value that defines if the user has admin permission for transcription. + ///i.e permissions to start/stop the transcription. + final bool admin; + + HMSTranscriptionPermission({required this.mode, required this.admin}); + + factory HMSTranscriptionPermission.fromMap(Map map) { + return HMSTranscriptionPermission( + mode: HMSTranscriptionModeValues.getHMSTranscriptionModeFromString( + map['mode']), + admin: map['admin']); + } +} diff --git a/packages/hmssdk_flutter/lib/src/service/platform_service.dart b/packages/hmssdk_flutter/lib/src/service/platform_service.dart index 29e51d085..701abb264 100644 --- a/packages/hmssdk_flutter/lib/src/service/platform_service.dart +++ b/packages/hmssdk_flutter/lib/src/service/platform_service.dart @@ -17,6 +17,7 @@ import 'package:hmssdk_flutter/hmssdk_flutter.dart'; import 'package:hmssdk_flutter/src/enum/hms_hls_playback_event_method.dart'; import 'package:hmssdk_flutter/src/enum/hms_key_change_listener_method.dart'; import 'package:hmssdk_flutter/src/enum/hms_logs_update_listener.dart'; +import 'package:hmssdk_flutter/src/enum/hms_transcription_listener_method.dart'; import 'package:hmssdk_flutter/src/model/hms_key_change_observer.dart'; import 'package:hmssdk_flutter/src/model/platform_method_response.dart'; @@ -55,6 +56,10 @@ abstract class PlatformService { static const EventChannel _whiteboardEventChannel = const EventChannel("whiteboard_event_channel"); + ///used to get transcription events + static const EventChannel _transcriptionEventChannel = + const EventChannel("transcription_event_channel"); + ///add meeting listeners. static List updateListeners = []; @@ -71,6 +76,8 @@ abstract class PlatformService { static HMSWhiteboardUpdateListener? _whiteboardListener; + static HMSTranscriptListener? _transcriptListener; + ///List for event Listener static List statsListeners = []; static bool isStartedListening = false; @@ -146,6 +153,14 @@ abstract class PlatformService { keyChangeObservers.add(hmsKeyChangeObserver); } + static void addTranscriptListener(HMSTranscriptListener transcriptListener) { + _transcriptListener = transcriptListener; + } + + static void removeTranscriptListener() { + _transcriptListener = null; + } + static Future removeKeyChangeObserver( HMSKeyChangeListener hmsKeyChangeListener) async { int index = keyChangeObservers.indexWhere((observer) => @@ -626,6 +641,27 @@ abstract class PlatformService { break; } }); + + _transcriptionEventChannel + .receiveBroadcastStream({'name': 'transcription'}).map((event) { + HMSTranscriptionListenerMethod method = + HMSTranscriptionListenerMethodValues + .getHMSTranscriptionListenerMethodFromString(event['event_name']); + var data = event['data']; + return HMSTranscriptionListenerMethodResponse(method: method, data: data); + }).listen((event) { + HMSTranscriptionListenerMethod method = event.method; + switch (method) { + case HMSTranscriptionListenerMethod.onTranscripts: + _transcriptListener?.onTranscripts( + transcriptions: (event.data) + .map((e) => HMSTranscription.fromMap(e)) + .toList()); + break; + case HMSTranscriptionListenerMethod.unknown: + break; + } + }); } static void notifyLogsUpdateListeners(