diff --git a/plugins/obs-webrtc/CMakeLists.txt b/plugins/obs-webrtc/CMakeLists.txt index 3b1f1a8a502f35..c886787528d5cc 100644 --- a/plugins/obs-webrtc/CMakeLists.txt +++ b/plugins/obs-webrtc/CMakeLists.txt @@ -10,15 +10,48 @@ endif() find_package(LibDataChannel 0.20 REQUIRED) find_package(CURL REQUIRED) +find_package( + FFmpeg REQUIRED + COMPONENTS avcodec + avfilter + avdevice + avutil + swscale + avformat + swresample) + +if(NOT TARGET OBS::media-playback) + add_subdirectory("${CMAKE_SOURCE_DIR}/deps/media-playback" "${CMAKE_BINARY_DIR}/deps/media-playback") +endif() add_library(obs-webrtc MODULE) add_library(OBS::webrtc ALIAS obs-webrtc) target_sources( - obs-webrtc PRIVATE # cmake-format: sortable - obs-webrtc.cpp webrtc-utils.h whep-source.cpp whep-source.h whip-output.cpp whip-output.h whip-service.cpp whip-service.h) + obs-webrtc + PRIVATE # cmake-format: sortable + obs-webrtc.cpp + webrtc-utils.h + whep-source.cpp + whep-source.h + whip-output.cpp + whip-output.h + whip-service.cpp + whip-service.h) -target_link_libraries(obs-webrtc PRIVATE OBS::libobs LibDataChannel::LibDataChannel CURL::libcurl) +target_link_libraries( + obs-webrtc + PRIVATE OBS::libobs + OBS::media-playback + LibDataChannel::LibDataChannel + CURL::libcurl + FFmpeg::avcodec + FFmpeg::avfilter + FFmpeg::avformat + FFmpeg::avdevice + FFmpeg::avutil + FFmpeg::swscale + FFmpeg::swresample) # cmake-format: off set_target_properties_obs(obs-webrtc PROPERTIES FOLDER plugins PREFIX "") diff --git a/plugins/obs-webrtc/cmake/legacy.cmake b/plugins/obs-webrtc/cmake/legacy.cmake index 3e0a004013e71a..6c67b8cf064757 100644 --- a/plugins/obs-webrtc/cmake/legacy.cmake +++ b/plugins/obs-webrtc/cmake/legacy.cmake @@ -8,14 +8,46 @@ endif() find_package(LibDataChannel 0.20 REQUIRED) find_package(CURL REQUIRED) +find_package( + FFmpeg REQUIRED + COMPONENTS avcodec + avfilter + avdevice + avutil + swscale + avformat + swresample) +if(NOT TARGET OBS::media-playback) + add_subdirectory("${CMAKE_SOURCE_DIR}/deps/media-playback" "${CMAKE_BINARY_DIR}/deps/media-playback") +endif() add_library(obs-webrtc MODULE) add_library(OBS::webrtc ALIAS obs-webrtc) -target_sources(obs-webrtc PRIVATE obs-webrtc.cpp whip-output.cpp whip-output.h whip-service.cpp whip-service.h - webrtc-utils.h) +target_sources( + obs-webrtc + PRIVATE obs-webrtc.cpp + whip-output.cpp + whip-output.h + whip-service.cpp + whip-service.h + whep-source.cpp + whep-source.h + webrtc-utils.h) -target_link_libraries(obs-webrtc PRIVATE OBS::libobs LibDataChannel::LibDataChannel CURL::libcurl) +target_link_libraries( + obs-webrtc + PRIVATE OBS::libobs + OBS::media-playback + LibDataChannel::LibDataChannel + CURL::libcurl + FFmpeg::avcodec + FFmpeg::avfilter + FFmpeg::avformat + FFmpeg::avdevice + FFmpeg::avutil + FFmpeg::swscale + FFmpeg::swresample) set_target_properties(obs-webrtc PROPERTIES FOLDER "plugins") diff --git a/plugins/obs-webrtc/data/locale/en-US.ini b/plugins/obs-webrtc/data/locale/en-US.ini index c94717ec884366..2fbdb910e806c8 100644 --- a/plugins/obs-webrtc/data/locale/en-US.ini +++ b/plugins/obs-webrtc/data/locale/en-US.ini @@ -1,4 +1,5 @@ Output.Name="WHIP Output" +Source.Name="WHEP Source" Service.Name="WHIP Service" Service.BearerToken="Bearer Token" diff --git a/plugins/obs-webrtc/obs-webrtc.cpp b/plugins/obs-webrtc/obs-webrtc.cpp index ebb2eb4fdc751e..94e5909b67337d 100644 --- a/plugins/obs-webrtc/obs-webrtc.cpp +++ b/plugins/obs-webrtc/obs-webrtc.cpp @@ -1,6 +1,7 @@ #include #include "whip-output.h" +#include "whep-source.h" #include "whip-service.h" OBS_DECLARE_MODULE() @@ -14,6 +15,7 @@ bool obs_module_load() { register_whip_output(); register_whip_service(); + register_whep_source(); return true; } diff --git a/plugins/obs-webrtc/whep-source.cpp b/plugins/obs-webrtc/whep-source.cpp new file mode 100644 index 00000000000000..0911095df7a7db --- /dev/null +++ b/plugins/obs-webrtc/whep-source.cpp @@ -0,0 +1,515 @@ +#include "whep-source.h" +#include "webrtc-utils.h" + +#define do_log(level, format, ...) \ + blog(level, "[obs-webrtc] [whep_source: '%s'] " format, \ + obs_source_get_name(source), ##__VA_ARGS__) + +const auto pli_interval = 500; + +const auto rtp_clockrate_video = 90000; +const auto rtp_clockrate_audio = 48000; + +const uint8_t naluTypeBitmask = 0x1F; +const uint8_t naluTypeSPS = 7; +const uint8_t naluTypePPS = 8; + +WHEPSource::WHEPSource(obs_data_t *settings, obs_source_t *source) + : source(source), + endpoint_url(), + resource_url(), + bearer_token(), + peer_connection(nullptr), + audio_track(nullptr), + video_track(nullptr), + running(false), + start_stop_mutex(), + start_stop_thread(), + last_frame(std::chrono::system_clock::now()), + last_audio_rtp_timestamp(0), + last_video_rtp_timestamp(0), + last_audio_pts(0), + last_video_pts(0) +{ + + this->video_av_codec_context = std::shared_ptr( + this->CreateVideoAVCodecDecoder(), + [](AVCodecContext *ctx) { avcodec_free_context(&ctx); }); + + this->audio_av_codec_context = std::shared_ptr( + this->CreateAudioAVCodecDecoder(), + [](AVCodecContext *ctx) { avcodec_free_context(&ctx); }); + + this->av_packet = std::shared_ptr( + av_packet_alloc(), [](AVPacket *pkt) { av_packet_free(&pkt); }); + this->av_frame = + std::shared_ptr(av_frame_alloc(), [](AVFrame *frame) { + av_frame_free(&frame); + }); + + Update(settings); +} + +WHEPSource::~WHEPSource() +{ + running = false; + + Stop(); + + std::lock_guard l(start_stop_mutex); + if (start_stop_thread.joinable()) + start_stop_thread.join(); +} + +void WHEPSource::Stop() +{ + std::lock_guard l(start_stop_mutex); + if (start_stop_thread.joinable()) + start_stop_thread.join(); + + start_stop_thread = std::thread(&WHEPSource::StopThread, this); +} + +void WHEPSource::StopThread() +{ + if (peer_connection != nullptr) { + peer_connection->close(); + peer_connection = nullptr; + audio_track = nullptr; + video_track = nullptr; + } + + SendDelete(); +} + +void WHEPSource::SendDelete() +{ + if (resource_url.empty()) { + do_log(LOG_DEBUG, + "No resource URL available, not sending DELETE"); + return; + } + + char error_buffer[CURL_ERROR_SIZE] = {}; + CURLcode curl_code; + auto status = send_delete(bearer_token, resource_url, &curl_code, + error_buffer); + if (status == webrtc_network_status::Success) { + do_log(LOG_DEBUG, + "Successfully performed DELETE request for resource URL"); + resource_url.clear(); + } else if (status == webrtc_network_status::DeleteFailed) { + do_log(LOG_WARNING, + "DELETE request for resource URL failed. Reason: %s", + error_buffer[0] ? error_buffer + : curl_easy_strerror(curl_code)); + } else if (status == webrtc_network_status::InvalidHTTPStatusCode) { + do_log(LOG_WARNING, + "DELETE request for resource URL returned non-200 Status Code"); + } +} + +obs_properties_t *WHEPSource::GetProperties() +{ + obs_properties_t *ppts = obs_properties_create(); + + obs_properties_set_flags(ppts, OBS_PROPERTIES_DEFER_UPDATE); + obs_properties_add_text(ppts, "endpoint_url", "URL", OBS_TEXT_DEFAULT); + obs_properties_add_text(ppts, "bearer_token", + obs_module_text("Service.BearerToken"), + OBS_TEXT_PASSWORD); + + return ppts; +} + +void WHEPSource::Update(obs_data_t *settings) +{ + endpoint_url = + std::string(obs_data_get_string(settings, "endpoint_url")); + bearer_token = + std::string(obs_data_get_string(settings, "bearer_token")); + + if (endpoint_url.empty() || bearer_token.empty()) { + return; + } + + std::lock_guard l(start_stop_mutex); + + if (start_stop_thread.joinable()) + start_stop_thread.join(); + + start_stop_thread = std::thread(&WHEPSource::StartThread, this); +} + +AVCodecContext *WHEPSource::CreateVideoAVCodecDecoder() +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); + if (!codec) { + throw std::runtime_error("Failed to find H264 codec"); + } + + AVCodecContext *codec_context = avcodec_alloc_context3(codec); + if (!codec_context) { + throw std::runtime_error("Failed to allocate codec context"); + } + + if (avcodec_open2(codec_context, codec, nullptr) < 0) { + throw std::runtime_error("Failed to open codec"); + } + + return codec_context; +} + +AVCodecContext *WHEPSource::CreateAudioAVCodecDecoder() +{ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_OPUS); + if (!codec) { + throw std::runtime_error("Failed to find Opus codec"); + } + + AVCodecContext *codec_context = avcodec_alloc_context3(codec); + if (!codec_context) { + throw std::runtime_error("Failed to allocate codec context"); + } + + if (avcodec_open2(codec_context, codec, nullptr) < 0) { + throw std::runtime_error("Failed to open codec"); + } + + return codec_context; +} + +static inline enum audio_format convert_sample_format(int f) +{ + switch (f) { + case AV_SAMPLE_FMT_U8: + return AUDIO_FORMAT_U8BIT; + case AV_SAMPLE_FMT_S16: + return AUDIO_FORMAT_16BIT; + case AV_SAMPLE_FMT_S32: + return AUDIO_FORMAT_32BIT; + case AV_SAMPLE_FMT_FLT: + return AUDIO_FORMAT_FLOAT; + case AV_SAMPLE_FMT_U8P: + return AUDIO_FORMAT_U8BIT_PLANAR; + case AV_SAMPLE_FMT_S16P: + return AUDIO_FORMAT_16BIT_PLANAR; + case AV_SAMPLE_FMT_S32P: + return AUDIO_FORMAT_32BIT_PLANAR; + case AV_SAMPLE_FMT_FLTP: + return AUDIO_FORMAT_FLOAT_PLANAR; + default:; + } + + return AUDIO_FORMAT_UNKNOWN; +} + +static inline enum speaker_layout convert_speaker_layout(uint8_t channels) +{ + switch (channels) { + case 0: + return SPEAKERS_UNKNOWN; + case 1: + return SPEAKERS_MONO; + case 2: + return SPEAKERS_STEREO; + case 3: + return SPEAKERS_2POINT1; + case 4: + return SPEAKERS_4POINT0; + case 5: + return SPEAKERS_4POINT1; + case 6: + return SPEAKERS_5POINT1; + case 8: + return SPEAKERS_7POINT1; + default: + return SPEAKERS_UNKNOWN; + } +} + +void WHEPSource::OnFrameAudio(rtc::binary msg, rtc::FrameInfo frame_info) +{ + auto pts = last_audio_pts; + if (this->last_audio_rtp_timestamp != 0) { + auto rtp_diff = + this->last_audio_rtp_timestamp - frame_info.timestamp; + pts += (rtp_diff / rtp_clockrate_audio); + } + + this->last_audio_rtp_timestamp = frame_info.timestamp; + + AVPacket *pkt = this->av_packet.get(); + pkt->data = reinterpret_cast(msg.data()); + pkt->size = static_cast(msg.size()); + + auto ret = avcodec_send_packet(this->audio_av_codec_context.get(), pkt); + if (ret < 0) { + return; + } + + AVFrame *av_frame = this->av_frame.get(); + + while (true) { + ret = avcodec_receive_frame(this->audio_av_codec_context.get(), + av_frame); + if (ret < 0) { + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + return; + } + + struct obs_source_audio audio = {}; + + audio.samples_per_sec = av_frame->sample_rate; + audio.speakers = + convert_speaker_layout(av_frame->ch_layout.nb_channels); + audio.format = convert_sample_format(av_frame->format); + audio.frames = av_frame->nb_samples; + audio.timestamp = pts; + + for (size_t i = 0; i < MAX_AV_PLANES; i++) { + audio.data[i] = av_frame->extended_data[i]; + } + + obs_source_output_audio(this->source, &audio); + } +} + +void WHEPSource::OnFrameVideo(rtc::binary msg, rtc::FrameInfo frame_info) +{ + auto pts = last_video_pts; + if (this->last_video_rtp_timestamp != 0) { + auto rtp_diff = + this->last_video_rtp_timestamp - frame_info.timestamp; + pts += (rtp_diff / rtp_clockrate_video); + } + + this->last_video_rtp_timestamp = frame_info.timestamp; + + AVPacket *pkt = this->av_packet.get(); + + auto naluType = uint8_t(msg.at(4)) & naluTypeBitmask; + if (naluType == naluTypeSPS || naluType == naluTypePPS) { + std::move(msg.begin(), msg.end(), + std::back_inserter(sps_and_pps)); + return; + } else if (this->sps_and_pps.size() != 0) { + std::copy(this->sps_and_pps.begin(), this->sps_and_pps.end(), + std::back_inserter(msg)); + this->sps_and_pps = std::vector{}; + } + + pkt->data = reinterpret_cast(msg.data()); + pkt->size = static_cast(msg.size()); + + auto ret = avcodec_send_packet(this->video_av_codec_context.get(), pkt); + if (ret < 0) { + return; + } + + AVFrame *av_frame = this->av_frame.get(); + + while (true) { + ret = avcodec_receive_frame(this->video_av_codec_context.get(), + av_frame); + if (ret < 0) { + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + return; + } + + struct obs_source_frame frame = {}; + + frame.format = VIDEO_FORMAT_I420; + frame.width = av_frame->width; + frame.height = av_frame->height; + frame.timestamp = pts; + frame.max_luminance = 0; + frame.trc = VIDEO_TRC_DEFAULT; + + video_format_get_parameters_for_format( + VIDEO_CS_DEFAULT, VIDEO_RANGE_DEFAULT, frame.format, + frame.color_matrix, frame.color_range_min, + frame.color_range_max); + + for (size_t i = 0; i < MAX_AV_PLANES; i++) { + frame.data[i] = av_frame->data[i]; + frame.linesize[i] = abs(av_frame->linesize[i]); + } + + obs_source_output_video(this->source, &frame); + last_frame = std::chrono::system_clock::now(); + } +} + +void WHEPSource::SetupPeerConnection() +{ + rtc::Configuration cfg; + +#if RTC_VERSION_MAJOR == 0 && RTC_VERSION_MINOR > 20 || RTC_VERSION_MAJOR > 1 + cfg.disableAutoGathering = true; +#endif + + peer_connection = std::make_shared(cfg); + + peer_connection->onStateChange([this](rtc::PeerConnection::State state) { + switch (state) { + case rtc::PeerConnection::State::New: + do_log(LOG_INFO, "PeerConnection state is now: New"); + break; + case rtc::PeerConnection::State::Connecting: + do_log(LOG_INFO, + "PeerConnection state is now: Connecting"); + break; + case rtc::PeerConnection::State::Connected: + do_log(LOG_INFO, + "PeerConnection state is now: Connected"); + break; + case rtc::PeerConnection::State::Disconnected: + do_log(LOG_INFO, + "PeerConnection state is now: Disconnected"); + break; + case rtc::PeerConnection::State::Failed: + do_log(LOG_INFO, "PeerConnection state is now: Failed"); + break; + case rtc::PeerConnection::State::Closed: + do_log(LOG_INFO, "PeerConnection state is now: Closed"); + break; + } + }); + + rtc::Description::Audio audioMedia( + "0", rtc::Description::Direction::RecvOnly); + audioMedia.addOpusCodec(111); + audio_track = peer_connection->addTrack(audioMedia); + + auto audio_depacketizer = std::make_shared(); + auto audio_session = std::make_shared(); + audio_session->addToChain(audio_depacketizer); + audio_track->setMediaHandler(audio_depacketizer); + audio_track->onFrame([&](rtc::binary msg, rtc::FrameInfo frame_info) { + this->OnFrameAudio(msg, frame_info); + }); + + rtc::Description::Video videoMedia( + "1", rtc::Description::Direction::RecvOnly); + videoMedia.addH264Codec(96); + video_track = peer_connection->addTrack(videoMedia); + + auto video_depacketizer = std::make_shared(); + auto video_session = std::make_shared(); + video_session->addToChain(video_depacketizer); + video_track->setMediaHandler(video_depacketizer); + video_track->onFrame([&](rtc::binary msg, rtc::FrameInfo frame_info) { + this->OnFrameVideo(msg, frame_info); + }); + + peer_connection->setLocalDescription(); +} + +void WHEPSource::StartThread() +{ + running = true; + SetupPeerConnection(); + + char error_buffer[CURL_ERROR_SIZE] = {}; + CURLcode curl_code; + auto status = send_offer(bearer_token, endpoint_url, peer_connection, + resource_url, &curl_code, error_buffer); + if (status != webrtc_network_status::Success) { + if (status == webrtc_network_status::ConnectFailed) { + do_log(LOG_WARNING, "Connect failed: %s", + error_buffer[0] ? error_buffer + : curl_easy_strerror(curl_code)); + } else if (status == + webrtc_network_status::InvalidHTTPStatusCode) { + do_log(LOG_ERROR, + "Connect failed: HTTP endpoint returned non-201 response code"); + } else if (status == webrtc_network_status::NoHTTPData) { + do_log(LOG_ERROR, + "Connect failed: No data returned from HTTP endpoint request"); + } else if (status == webrtc_network_status::NoLocationHeader) { + do_log(LOG_ERROR, + "WHEP server did not provide a resource URL via the Location header"); + } else if (status == + webrtc_network_status::FailedToBuildResourceURL) { + do_log(LOG_ERROR, "Failed to build Resource URL"); + } else if (status == + webrtc_network_status::InvalidLocationHeader) { + do_log(LOG_ERROR, + "WHEP server provided a invalid resource URL via the Location header"); + } else if (status == webrtc_network_status::InvalidAnswer) { + do_log(LOG_WARNING, + "WHIP server responded with invalid SDP: %s", + error_buffer); + } else if (status == + webrtc_network_status::SetRemoteDescriptionFailed) { + do_log(LOG_WARNING, + "Failed to set remote description: %s", + error_buffer); + } + + peer_connection->close(); + return; + } + + do_log(LOG_DEBUG, "WHEP Resource URL is: %s", resource_url.c_str()); +} + +void WHEPSource::MaybeSendPLI() +{ + auto time_since_frame = + std::chrono::system_clock::now() - last_frame.load(); + + if (std::chrono::duration_cast( + time_since_frame) + .count() < pli_interval) { + return; + } + + auto time_since_pli = std::chrono::system_clock::now() - last_pli; + if (std::chrono::duration_cast( + time_since_pli) + .count() < pli_interval) { + return; + } + + if (video_track != nullptr) { + video_track->requestKeyframe(); + } + + last_pli = std::chrono::system_clock::now(); +} + +void register_whep_source() +{ + struct obs_source_info info = {}; + + info.id = "whep_source"; + info.type = OBS_SOURCE_TYPE_INPUT; + info.output_flags = OBS_SOURCE_ASYNC_VIDEO | OBS_SOURCE_AUDIO | + OBS_SOURCE_DO_NOT_DUPLICATE; + info.get_name = [](void *) -> const char * { + return obs_module_text("Source.Name"); + }; + info.create = [](obs_data_t *settings, obs_source_t *source) -> void * { + return new WHEPSource(settings, source); + }; + info.destroy = [](void *priv_data) { + delete static_cast(priv_data); + }; + info.get_properties = [](void *priv_data) -> obs_properties_t * { + return static_cast(priv_data)->GetProperties(); + }; + info.update = [](void *priv_data, obs_data_t *settings) { + static_cast(priv_data)->Update(settings); + }; + info.video_tick = [](void *priv_data, float) { + static_cast(priv_data)->MaybeSendPLI(); + }; + + obs_register_source(&info); +} diff --git a/plugins/obs-webrtc/whep-source.h b/plugins/obs-webrtc/whep-source.h new file mode 100644 index 00000000000000..927b3a80d31595 --- /dev/null +++ b/plugins/obs-webrtc/whep-source.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +extern "C" { +#include "libavcodec/avcodec.h" +} + +#include + +class WHEPSource { +public: + WHEPSource(obs_data_t *settings, obs_source_t *source); + ~WHEPSource(); + + obs_properties_t *GetProperties(); + void Update(obs_data_t *settings); + void MaybeSendPLI(); + + std::atomic last_frame; + std::chrono::system_clock::time_point last_pli; + +private: + bool Init(); + void SetupPeerConnection(); + bool Connect(); + void StartThread(); + void SendDelete(); + void Stop(); + void StopThread(); + + AVCodecContext *CreateVideoAVCodecDecoder(); + AVCodecContext *CreateAudioAVCodecDecoder(); + + void OnFrameAudio(rtc::binary msg, rtc::FrameInfo frame_info); + void OnFrameVideo(rtc::binary msg, rtc::FrameInfo frame_info); + + obs_source_t *source; + + std::string endpoint_url; + std::string resource_url; + std::string bearer_token; + + std::shared_ptr peer_connection; + std::shared_ptr audio_track; + std::shared_ptr video_track; + + std::shared_ptr video_av_codec_context; + std::shared_ptr audio_av_codec_context; + std::shared_ptr av_packet; + std::shared_ptr av_frame; + + std::atomic running; + std::mutex start_stop_mutex; + std::thread start_stop_thread; + + std::vector sps_and_pps; + + uint64_t last_audio_rtp_timestamp, last_video_rtp_timestamp; + uint64_t last_audio_pts, last_video_pts; +}; + +void register_whep_source();