From 3b7b72b25a4d70fdbf3b0c7efd1f322d25053958 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Tue, 11 Apr 2023 09:24:47 +0800 Subject: [PATCH] Remove SDP munging for P2P mode. This change replaces SDP munging with setCodecPreference for P2P mode. Changes of OWT APIs are needed to fully support setCodecPreference. For example, OWT APIs allow different bitrate settings for different codecs, which is not supported by WebRTC, so only the first codec's bitrate setting takes effect at this time. --- talk/owt/sdk/base/peerconnectionchannel.cc | 14 +- talk/owt/sdk/base/peerconnectionchannel.h | 13 +- .../base/peerconnectiondependencyfactory.cc | 17 ++ .../base/peerconnectiondependencyfactory.h | 6 +- .../sdk/include/cpp/owt/base/commontypes.h | 7 +- talk/owt/sdk/p2p/p2ppeerconnectionchannel.cc | 149 +++++++++++++++--- 6 files changed, 168 insertions(+), 38 deletions(-) diff --git a/talk/owt/sdk/base/peerconnectionchannel.cc b/talk/owt/sdk/base/peerconnectionchannel.cc index 21d78020a..0a4438f8c 100644 --- a/talk/owt/sdk/base/peerconnectionchannel.cc +++ b/talk/owt/sdk/base/peerconnectionchannel.cc @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "talk/owt/sdk/base/peerconnectionchannel.h" #include -#include "talk/owt/sdk/base/sdputils.h" #include "webrtc/api/peer_connection_interface.h" #include "webrtc/rtc_base/logging.h" #include "webrtc/rtc_base/thread.h" @@ -83,16 +82,17 @@ void PeerConnectionChannel::ApplyBitrateSettings() { return; } -void PeerConnectionChannel::AddTransceiver( +webrtc::RTCErrorOr> +PeerConnectionChannel::AddTransceiver( rtc::scoped_refptr track, const webrtc::RtpTransceiverInit& init) { - peer_connection_->AddTransceiver(track, init); + return peer_connection_->AddTransceiver(track, init); } -void PeerConnectionChannel::AddTransceiver( - cricket::MediaType media_type, - const webrtc::RtpTransceiverInit& init) { - peer_connection_->AddTransceiver(media_type, init); +webrtc::RTCErrorOr> +PeerConnectionChannel::AddTransceiver(cricket::MediaType media_type, + const webrtc::RtpTransceiverInit& init) { + return peer_connection_->AddTransceiver(media_type, init); } const webrtc::SessionDescriptionInterface* diff --git a/talk/owt/sdk/base/peerconnectionchannel.h b/talk/owt/sdk/base/peerconnectionchannel.h index 14e550b15..5535b2716 100644 --- a/talk/owt/sdk/base/peerconnectionchannel.h +++ b/talk/owt/sdk/base/peerconnectionchannel.h @@ -41,11 +41,14 @@ class PeerConnectionChannel : public webrtc::PeerConnectionObserver, // message to PeerConnectionChannel. virtual void CreateOffer() = 0; virtual void CreateAnswer() = 0; - virtual void AddTransceiver( - rtc::scoped_refptr track, - const webrtc::RtpTransceiverInit& init); - virtual void AddTransceiver(cricket::MediaType media_type, - const webrtc::RtpTransceiverInit& init); + virtual webrtc::RTCErrorOr< + rtc::scoped_refptr> + AddTransceiver(rtc::scoped_refptr track, + const webrtc::RtpTransceiverInit& init); + virtual webrtc::RTCErrorOr< + rtc::scoped_refptr> + AddTransceiver(cricket::MediaType media_type, + const webrtc::RtpTransceiverInit& init); // PeerConnectionObserver virtual void OnStateChange(webrtc::StatsReport::StatsType state_changed) {} virtual void OnSignalingChange( diff --git a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc index d9ccdbc60..658e38cb0 100644 --- a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc +++ b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc @@ -373,5 +373,22 @@ scoped_refptr PeerConnectionDependencyFactory:: GlobalConfiguration::GetAudioFrameGenerator()); } +std::unique_ptr +PeerConnectionDependencyFactory::GetSenderCapabilities( + const std::string& kind) { + if (kind == "audio") { + return pc_thread_->BlockingCall([this] { + return std::make_unique( + pc_factory_->GetRtpSenderCapabilities(cricket::MEDIA_TYPE_AUDIO)); + }); + } else if (kind == "video") { + return pc_thread_->BlockingCall([this] { + return std::make_unique( + pc_factory_->GetRtpSenderCapabilities(cricket::MEDIA_TYPE_VIDEO)); + }); + } + return nullptr; +} + } // namespace base } // namespace owt diff --git a/talk/owt/sdk/base/peerconnectiondependencyfactory.h b/talk/owt/sdk/base/peerconnectiondependencyfactory.h index 45641f8fb..71ae75ba1 100644 --- a/talk/owt/sdk/base/peerconnectiondependencyfactory.h +++ b/talk/owt/sdk/base/peerconnectiondependencyfactory.h @@ -35,6 +35,7 @@ class PeerConnectionThread : public rtc::Thread { // Object factory for WebRTC PeerConnections. class PeerConnectionDependencyFactory : public rtc::RefCountInterface { public: + ~PeerConnectionDependencyFactory() override; // Get a PeerConnectionDependencyFactory instance. It doesn't create a new // instance. It always return the same instance. static PeerConnectionDependencyFactory* Get(); @@ -57,9 +58,12 @@ class PeerConnectionDependencyFactory : public rtc::RefCountInterface { // Returns current |pc_factory_|. rtc::scoped_refptr PeerConnectionFactory() const; + std::unique_ptr GetSenderCapabilities( + const std::string& kind); + // Returns |signaling_thread_| for testing. rtc::Thread* SignalingThreadForTesting(); - ~PeerConnectionDependencyFactory() override; + protected: explicit PeerConnectionDependencyFactory(); virtual const rtc::scoped_refptr& diff --git a/talk/owt/sdk/include/cpp/owt/base/commontypes.h b/talk/owt/sdk/include/cpp/owt/base/commontypes.h index d18e463e3..3b7091040 100644 --- a/talk/owt/sdk/include/cpp/owt/base/commontypes.h +++ b/talk/owt/sdk/include/cpp/owt/base/commontypes.h @@ -129,7 +129,8 @@ struct OWT_EXPORT RtpEncodingParameters { NetworkPriority priority = NetworkPriority::kDefault; }; -/// Audio encoding parameters. +/// Audio encoding parameters. This API will be changed in the future to align +/// with WebRTC APIs. struct OWT_EXPORT AudioEncodingParameters { explicit AudioEncodingParameters(); AudioEncodingParameters(const AudioCodecParameters& codec_param, @@ -148,11 +149,13 @@ struct OWT_EXPORT VideoCodecParameters { VideoCodecParameters(const VideoCodec& codec, const std::string& profile); virtual ~VideoCodecParameters(); VideoCodec name; + // profile is ignore at this time. std::string profile; }; /// Video encoding parameters. Used to specify the video encoding settings when -/// publishing the video. +/// publishing the video. This API will be changed in the future to align with +/// WebRTC APIs. struct OWT_EXPORT VideoEncodingParameters { explicit VideoEncodingParameters(); /// Construct an instance of VideoEncodingParameters diff --git a/talk/owt/sdk/p2p/p2ppeerconnectionchannel.cc b/talk/owt/sdk/p2p/p2ppeerconnectionchannel.cc index ec8f1a72c..58f7ccd2a 100644 --- a/talk/owt/sdk/p2p/p2ppeerconnectionchannel.cc +++ b/talk/owt/sdk/p2p/p2ppeerconnectionchannel.cc @@ -5,7 +5,6 @@ #include #include "talk/owt/sdk/base/eventtrigger.h" #include "talk/owt/sdk/base/functionalobserver.h" -#include "talk/owt/sdk/base/sdputils.h" #include "talk/owt/sdk/base/sysinfo.h" #include "talk/owt/sdk/p2p/p2ppeerconnectionchannel.h" #include "webrtc/rtc_base/logging.h" @@ -14,6 +13,23 @@ using namespace rtc; namespace owt { namespace p2p { + +static const std:: + unordered_map + audio_codec_names = {{owt::base::AudioCodec::kOpus, "OPUS"}, + {owt::base::AudioCodec::kIsac, "ISAC"}, + {owt::base::AudioCodec::kG722, "G722"}, + {owt::base::AudioCodec::kPcmu, "PCMU"}, + {owt::base::AudioCodec::kIlbc, "ILBC"}, + {owt::base::AudioCodec::kPcma, "PCMA"}}; +static const std:: + unordered_map + video_codec_names = {{owt::base::VideoCodec::kVp8, "VP8"}, + {owt::base::VideoCodec::kVp9, "VP9"}, + {owt::base::VideoCodec::kH264, "H264"}, + {owt::base::VideoCodec::kH265, "H265"}, + {owt::base::VideoCodec::kAv1, "AV1"}}; + using std::string; enum P2PPeerConnectionChannel::SessionState : int { kSessionStateReady = 1, // Indicate the channel is ready. This is the initial state. @@ -636,16 +652,6 @@ void P2PPeerConnectionChannel::OnSignalingChange( RTC_LOG(LS_ERROR) << "Error parsing local description."; RTC_DCHECK(false); } - std::vector audio_codecs; - for (auto& audio_enc_param : configuration_.audio) { - audio_codecs.push_back(audio_enc_param.codec.name); - } - sdp_string = SdpUtils::SetPreferAudioCodecs(sdp_string, audio_codecs); - std::vector video_codecs; - for (auto& video_enc_param : configuration_.video) { - video_codecs.push_back(video_enc_param.codec.name); - } - sdp_string = SdpUtils::SetPreferVideoCodecs(sdp_string, video_codecs); std::unique_ptr new_desc( webrtc::CreateSessionDescription(pending_remote_sdp_->type(), sdp_string, nullptr)); @@ -827,16 +833,6 @@ void P2PPeerConnectionChannel::OnCreateSessionDescriptionSuccess( RTC_LOG(LS_ERROR) << "Error parsing local description."; RTC_DCHECK(false); } - std::vector audio_codecs; - for (auto& audio_enc_param : configuration_.audio) { - audio_codecs.push_back(audio_enc_param.codec.name); - } - sdp_string = SdpUtils::SetPreferAudioCodecs(sdp_string, audio_codecs); - std::vector video_codecs; - for (auto& video_enc_param : configuration_.video) { - video_codecs.push_back(video_enc_param.codec.name); - } - sdp_string = SdpUtils::SetPreferVideoCodecs(sdp_string, video_codecs); webrtc::SessionDescriptionInterface* new_desc( webrtc::CreateSessionDescription(desc->type(), sdp_string, nullptr)); peer_connection_->SetLocalDescription(observer.get(), new_desc); @@ -1173,16 +1169,123 @@ void P2PPeerConnectionChannel::DrainPendingStreams() { track_info[kTrackIdKey] = track->id(); track_info[kTrackSourceKey] = audio_track_source; track_sources.append(track_info); - peer_connection_->AddTrack(track, {media_stream->id()}); + + webrtc::RtpTransceiverInit init; + init.direction = webrtc::RtpTransceiverDirection::kSendRecv; + init.stream_ids.push_back(media_stream->id()); + if (configuration_.audio.size() > 0) { + // OWT APIs allow different bitrate settings for different codecs. + // However, this is not supported by WebRTC. We take the first + // codec's setting here. Consider to change OWT API in the future. + const auto& audio_encoding_paramters = + configuration_.audio[0].rtp_encoding_parameters; + std::vector + rtp_encoding_parameters_list; + rtp_encoding_parameters_list.resize(audio_encoding_paramters.size()); + std::transform( + audio_encoding_paramters.begin(), audio_encoding_paramters.end(), + rtp_encoding_parameters_list.begin(), + [](const RtpEncodingParameters& p) { + webrtc::RtpEncodingParameters encoding_paramters; + encoding_paramters.active = p.active; + if (p.max_bitrate_bps != 0) { + encoding_paramters.max_bitrate_bps = p.max_bitrate_bps; + } + if (p.max_framerate != 0) { + encoding_paramters.max_framerate = p.max_framerate; + } + if (p.rid != "") { + encoding_paramters.rid = p.rid; + } + return encoding_paramters; + }); + init.send_encodings = rtp_encoding_parameters_list; + } + auto transceiver = AddTransceiver(track, init); + if (!configuration_.audio.empty() && transceiver.ok()) { + std::vector codecs; + auto capabilities = + PeerConnectionDependencyFactory::Get()->GetSenderCapabilities( + "audio"); + for (const auto& audio : configuration_.audio) { + for (auto& c : capabilities->codecs) { + if (c.name != audio_codec_names.at(audio.codec.name)) { + continue; + } + if (audio.codec.channel_count != 0 && + c.num_channels != + static_cast(audio.codec.channel_count)) { + continue; + } + codecs.push_back(c); + } + } + transceiver.value()->SetCodecPreferences(codecs); + } } for (const auto& track : media_stream->GetVideoTracks()) { + RTC_LOG(LS_INFO)<<"GetVideoTracks(), config empty: "<id()); stream_sources[kStreamVideoSourceKey] = video_track_source; track_info[kTrackIdKey] = track->id(); track_info[kTrackSourceKey] = video_track_source; track_sources.append(track_info); - peer_connection_->AddTrack(track, {media_stream->id()}); + + webrtc::RtpTransceiverInit init; + init.direction = webrtc::RtpTransceiverDirection::kSendRecv; + init.stream_ids.push_back(media_stream->id()); + if (!configuration_.video.empty()) { + // OWT APIs allow different bitrate settings for different codecs. + // However, this is not supported by WebRTC. We take the first + // codec's setting here. Consider to change OWT API in the future. + const auto& video_encoding_paramters = + configuration_.video[0].rtp_encoding_parameters; + std::vector + rtp_encoding_parameters_list; + rtp_encoding_parameters_list.resize(video_encoding_paramters.size()); + std::transform( + video_encoding_paramters.begin(), video_encoding_paramters.end(), + rtp_encoding_parameters_list.begin(), + [](const RtpEncodingParameters& p) { + webrtc::RtpEncodingParameters encoding_paramters; + encoding_paramters.active = p.active; + if (p.max_bitrate_bps != 0) { + encoding_paramters.max_bitrate_bps = p.max_bitrate_bps; + } + if (p.max_framerate != 0) { + encoding_paramters.max_framerate = p.max_framerate; + } + if (p.rid != "") { + encoding_paramters.rid = p.rid; + } + if (p.num_temporal_layers != 0) { + encoding_paramters.num_temporal_layers = + p.num_temporal_layers; + } + if (p.scale_resolution_down_by != 0) { + encoding_paramters.scale_resolution_down_by = + p.scale_resolution_down_by; + } + return encoding_paramters; + }); + init.send_encodings = rtp_encoding_parameters_list; + } + auto transceiver = AddTransceiver(track, init); + if (!configuration_.video.empty() && transceiver.ok()) { + std::vector codecs; + auto capabilities = + PeerConnectionDependencyFactory::Get()->GetSenderCapabilities( + "video"); + for (const auto& video : configuration_.video) { + for (auto& c : capabilities->codecs) { + if (c.name == video_codec_names.at(video.codec.name)) { + codecs.push_back(c); + } + } + } + transceiver.value()->SetCodecPreferences(codecs); + } } // The second signaling message of track sources to remote peer. Json::Value json_track_sources;