From e4dee511cab0a98e802e2c0cc12b3d8b9b90a8df Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 18 Mar 2024 17:47:56 -0700 Subject: [PATCH 01/21] Use LL::WorkQueue to handle transitions from llwebrtc threads to the main thread --- indra/llwebrtc/llwebrtc.cpp | 2 +- indra/newview/llvoicewebrtc.cpp | 151 ++++++++++++++++++++++++++------ indra/newview/llvoicewebrtc.h | 26 +++--- 3 files changed, 143 insertions(+), 36 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index b7501bd0e0..6400f807fd 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -580,7 +580,7 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - mWebRTCImpl->SignalingBlockingCall( + mWebRTCImpl->PostSignalingTask( [this]() { if (mPeerConnection) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index cae7154bc2..1441e4e305 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -1,5 +1,5 @@ /** - * @file LLWebRTCVoiceClient.cpp + * @file llvoicewebrtc.cpp * @brief Implementation of LLWebRTCVoiceClient class which is the interface to the voice client process. * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ @@ -585,8 +585,19 @@ void LLWebRTCVoiceClient::setDevicesListUpdated(bool state) mDevicesListUpdated = state; } -void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceList &render_devices, - const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices) +// the singleton 'this' pointer will outlive the work queue. +void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceList& render_devices, + const llwebrtc::LLWebRTCVoiceDeviceList& capture_devices) +{ + LL::WorkQueue::postMaybe(mMainQueue, + [=] + { + OnDevicesChangedImpl(render_devices, capture_devices); + }); +} + +void LLWebRTCVoiceClient::OnDevicesChangedImpl(const llwebrtc::LLWebRTCVoiceDeviceList &render_devices, + const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices) { std::string inputDevice = gSavedSettings.getString("VoiceInputAudioDevice"); std::string outputDevice = gSavedSettings.getString("VoiceOutputAudioDevice"); @@ -1412,6 +1423,7 @@ void LLWebRTCVoiceClient::setVoiceEnabled(bool enabled) updatePosition(); if (!mIsCoroutineActive) { + mMainQueue = LL::WorkQueue::getInstance("mainloop"); LLCoros::instance().launch("LLWebRTCVoiceClient::voiceConnectionCoro", boost::bind(&LLWebRTCVoiceClient::voiceConnectionCoro, LLWebRTCVoiceClient::getInstance())); } @@ -2000,6 +2012,7 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID ®ionID, const s mWebRTCPeerConnectionInterface = llwebrtc::newPeerConnection(); mWebRTCPeerConnectionInterface->setSignalingObserver(this); + mMainQueue = LL::WorkQueue::getInstance("mainloop"); } LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() @@ -2026,7 +2039,14 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() // negotiated, updates about the best connectivity paths may trickle in. These need to be // sent to the Secondlife WebRTC server via the simulator so that both sides have a clear // view of the network environment. + +// callback from llwebrtc void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state) +{ + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnIceGatheringStateImpl(state); }); +} + +void LLVoiceWebRTCConnection::OnIceGatheringStateImpl(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state) { LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL; @@ -2034,13 +2054,11 @@ void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObs { case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE: { - LLMutexLock lock(&mVoiceStateMutex); mIceCompleted = true; break; } case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW: { - LLMutexLock lock(&mVoiceStateMutex); mIceCompleted = false; } default: @@ -2048,9 +2066,14 @@ void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObs } } -void LLVoiceWebRTCConnection::OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate &candidate) +// callback from llwebrtc +void LLVoiceWebRTCConnection::OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate& candidate) +{ + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnIceCandidateImpl(candidate); }); +} + +void LLVoiceWebRTCConnection::OnIceCandidateImpl(const llwebrtc::LLWebRTCIceCandidate &candidate) { - LLMutexLock lock(&mVoiceStateMutex); mIceCandidates.push_back(candidate); } @@ -2182,10 +2205,24 @@ void LLVoiceWebRTCConnection::processIceUpdates() // and is passed to the simulator via a CAP, which then passes // it on to the Secondlife WebRTC server. +// callback from llwebrtc void LLVoiceWebRTCConnection::OnOfferAvailable(const std::string &sdp) { + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnOfferAvailableImpl(sdp); }); +} + +// +// The LLWebRTCVoiceConnection object will not be deleted +// before the webrtc connection itself is shut down, so +// we shouldn't be getting this callback on a nonexistant +// this pointer. +void LLVoiceWebRTCConnection::OnOfferAvailableImpl(const std::string &sdp) +{ + if (mShutDown) + { + return; + } LL_DEBUGS("Voice") << "On Offer Available." << LL_ENDL; - LLMutexLock lock(&mVoiceStateMutex); mChannelSDP = sdp; if (mVoiceConnectionState == VOICE_STATE_WAIT_FOR_SESSION_START) { @@ -2193,15 +2230,42 @@ void LLVoiceWebRTCConnection::OnOfferAvailable(const std::string &sdp) } } -// Notifications from the webrtc library. -void LLVoiceWebRTCConnection::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) +// callback from llwebrtc +void LLVoiceWebRTCConnection::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface* audio_interface) { + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnAudioEstablishedImpl(audio_interface); }); +} + +// +// The LLWebRTCVoiceConnection object will not be deleted +// before the webrtc connection itself is shut down, so +// we shouldn't be getting this callback on a nonexistant +// this pointer. +// nor should audio_interface be invalid if the LLWebRTCVoiceConnection +// is shut down. +void LLVoiceWebRTCConnection::OnAudioEstablishedImpl(llwebrtc::LLWebRTCAudioInterface *audio_interface) +{ + if (mShutDown) + { + return; + } LL_DEBUGS("Voice") << "On AudioEstablished." << LL_ENDL; mWebRTCAudioInterface = audio_interface; setVoiceConnectionState(VOICE_STATE_SESSION_ESTABLISHED); } +// callback from llwebrtc void LLVoiceWebRTCConnection::OnRenegotiationNeeded() +{ + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnRenegotiationNeededImpl(); }); +} + +// +// The LLWebRTCVoiceConnection object will not be deleted +// before the webrtc connection itself is shut down, so +// we shouldn't be getting this callback on a nonexistant +// this pointer. +void LLVoiceWebRTCConnection::OnRenegotiationNeededImpl() { LL_DEBUGS("Voice") << "Voice channel requires renegotiation." << LL_ENDL; if (!mShutDown) @@ -2210,7 +2274,13 @@ void LLVoiceWebRTCConnection::OnRenegotiationNeeded() } } +// callback from llwebrtc void LLVoiceWebRTCConnection::OnPeerConnectionShutdown() +{ + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnPeerConnectionShutdownImpl(); }); +} + +void LLVoiceWebRTCConnection::OnPeerConnectionShutdownImpl() { setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); mOutstandingRequests--; // shut down is an async call which is handled on a webrtc thread. @@ -2410,7 +2480,6 @@ bool LLVoiceWebRTCSpatialConnection::requestVoiceConnection() LLSD jsep; jsep["type"] = "offer"; { - LLMutexLock lock(&mVoiceStateMutex); jsep["sdp"] = mChannelSDP; } body["jsep"] = jsep; @@ -2620,7 +2689,6 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() case VOICE_STATE_SESSION_EXIT: { { - LLMutexLock lock(&mVoiceStateMutex); if (!mShutDown) { mVoiceConnectionState = VOICE_STATE_START_SESSION; @@ -2649,19 +2717,35 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() } // Data has been received on the webrtc data channel -void LLVoiceWebRTCConnection::OnDataReceived(const std::string &data, bool binary) +// incoming data will be a json structure (if it's not binary.) We may pack +// binary for size reasons. Most of the keys in the json objects are +// single or double characters for size reasons. +// The primary element is: +// An object where each key is an agent id. (in the future, we may allow +// integer indices into an agentid list, populated on join commands. For size. +// Each key will point to a json object with keys identifying what's updated. +// 'p' - audio source power (level/volume) (int8 as int) +// 'j' - object of join data (currently only a boolean 'p' marking a primary participant) +// 'l' - boolean, always true if exists. +// 'v' - boolean - voice activity has been detected. + +// llwebrtc callback +void LLVoiceWebRTCConnection::OnDataReceived(const std::string& data, bool binary) { - // incoming data will be a json structure (if it's not binary.) We may pack - // binary for size reasons. Most of the keys in the json objects are - // single or double characters for size reasons. - // The primary element is: - // An object where each key is an agent id. (in the future, we may allow - // integer indices into an agentid list, populated on join commands. For size. - // Each key will point to a json object with keys identifying what's updated. - // 'p' - audio source power (level/volume) (int8 as int) - // 'j' - object of join data (currently only a boolean 'p' marking a primary participant) - // 'l' - boolean, always true if exists. - // 'v' - boolean - voice activity has been detected. + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnDataReceivedImpl(data, binary); }); +} + +// +// The LLWebRTCVoiceConnection object will not be deleted +// before the webrtc connection itself is shut down, so +// we shouldn't be getting this callback on a nonexistant +// this pointer. +void LLVoiceWebRTCConnection::OnDataReceivedImpl(const std::string &data, bool binary) +{ + if (mShutDown) + { + return; + } if (binary) { @@ -2769,8 +2853,26 @@ void LLVoiceWebRTCConnection::OnDataReceived(const std::string &data, bool binar } } +// llwebrtc callback void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) { + LL::WorkQueue::postMaybe(mMainQueue, [=] { OnDataChannelReadyImpl(data_interface); }); +} + +// +// The LLWebRTCVoiceConnection object will not be deleted +// before the webrtc connection itself is shut down, so +// we shouldn't be getting this callback on a nonexistant +// this pointer. +// nor should data_interface be invalid if the LLWebRTCVoiceConnection +// is shut down. +void LLVoiceWebRTCConnection::OnDataChannelReadyImpl(llwebrtc::LLWebRTCDataInterface *data_interface) +{ + if (mShutDown) + { + return; + } + if (data_interface) { mWebRTCDataInterface = data_interface; @@ -2894,7 +2996,6 @@ bool LLVoiceWebRTCAdHocConnection::requestVoiceConnection() LLSD jsep; jsep["type"] = "offer"; { - LLMutexLock lock(&mVoiceStateMutex); jsep["sdp"] = mChannelSDP; } body["jsep"] = jsep; diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index b26bea27ce..b0c214dfd5 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -225,7 +225,8 @@ public: void OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceList &render_devices, const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices) override; //@} - + void OnDevicesChangedImpl(const llwebrtc::LLWebRTCVoiceDeviceList &render_devices, + const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices); struct participantState { @@ -445,6 +446,8 @@ private: /// Clean up objects created during a voice session. void cleanUp(); + LL::WorkQueue::weak_t mMainQueue; + bool mTuningMode; F32 mTuningMicGain; int mTuningSpeakerVolume; @@ -588,6 +591,13 @@ class LLVoiceWebRTCConnection : void OnPeerConnectionShutdown() override; //@} + void OnIceGatheringStateImpl(EIceGatheringState state); + void OnIceCandidateImpl(const llwebrtc::LLWebRTCIceCandidate &candidate); + void OnOfferAvailableImpl(const std::string &sdp); + void OnRenegotiationNeededImpl(); + void OnAudioEstablishedImpl(llwebrtc::LLWebRTCAudioInterface *audio_interface); + void OnPeerConnectionShutdownImpl(); + ///////////////////////// /// @name Data Notification /// LLWebRTCDataObserver @@ -596,6 +606,9 @@ class LLVoiceWebRTCConnection : void OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) override; //@} + void OnDataReceivedImpl(const std::string &data, bool binary); + void OnDataChannelReadyImpl(llwebrtc::LLWebRTCDataInterface *data_interface); + void sendJoin(); void sendData(const std::string &data); @@ -618,7 +631,6 @@ class LLVoiceWebRTCConnection : void shutDown() { - LLMutexLock lock(&mVoiceStateMutex); mShutDown = true; } @@ -644,11 +656,10 @@ class LLVoiceWebRTCConnection : } EVoiceConnectionState; EVoiceConnectionState mVoiceConnectionState; - LLMutex mVoiceStateMutex; + LL::WorkQueue::weak_t mMainQueue; + void setVoiceConnectionState(EVoiceConnectionState new_voice_connection_state) { - LLMutexLock lock(&mVoiceStateMutex); - if (new_voice_connection_state & VOICE_STATE_SESSION_STOPPING) { // the new state is shutdown or restart. @@ -666,11 +677,6 @@ class LLVoiceWebRTCConnection : } EVoiceConnectionState getVoiceConnectionState() { - if (mVoiceStateMutex.isLocked()) - { - LL_WARNS("Voice") << "LOCKED." << LL_ENDL; - } - LLMutexLock lock(&mVoiceStateMutex); return mVoiceConnectionState; } From 5bc92b8031e9a6258bab8b24d42090a495cec6e5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Mar 2024 15:23:43 -0700 Subject: [PATCH 02/21] Simplify workqueue calls. Fix issue with webrtc blocking on destruction. --- indra/llwebrtc/llwebrtc.cpp | 62 +++--- indra/llwebrtc/llwebrtc.h | 3 - indra/llwebrtc/llwebrtc_impl.h | 16 +- indra/newview/llvoicewebrtc.cpp | 360 +++++++++++++------------------- indra/newview/llvoicewebrtc.h | 21 +- 5 files changed, 199 insertions(+), 263 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 6400f807fd..283b94dd3c 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -562,8 +562,10 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn // Most peer connection (signaling) happens on // the signaling thread. -LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : +LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() : mWebRTCImpl(nullptr), + mClosing(false), + mPeerConnection(nullptr), mMute(false), mAnswerReceived(false) { @@ -580,13 +582,24 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { + rtc::scoped_refptr connection; + mPeerConnection.swap(connection); + rtc::scoped_refptr dataChannel; + mDataChannel.swap(dataChannel); + rtc::scoped_refptr localStream; + mLocalStream.swap(localStream); + mWebRTCImpl->PostSignalingTask( - [this]() + [=]() { - if (mPeerConnection) + if (connection) { - mPeerConnection->Close(); - mPeerConnection = nullptr; + connection->Close(); + } + if (dataChannel) + { + dataChannel->UnregisterObserver(); + dataChannel->Close(); } }); } @@ -710,24 +723,9 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() bool LLWebRTCPeerConnectionImpl::shutdownConnection() { - if (mPeerConnection) - { - mWebRTCImpl->PostSignalingTask( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - for (auto &observer : mSignalingObserverList) - { - observer->OnPeerConnectionShutdown(); - } - }); - return true; - } - return false; + mClosing = true; + terminate(); + return true; } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) @@ -1057,14 +1055,16 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * } RTC_LOG(LS_INFO) << __FUNCTION__ << " Local SDP: " << sdp_mangled_stream.str(); - + std::string mangled_sdp = sdp_mangled_stream.str(); for (auto &observer : mSignalingObserverList) { - observer->OnOfferAvailable(sdp_mangled_stream.str()); + observer->OnOfferAvailable(mangled_sdp); } + + mPeerConnection->SetLocalDescription(std::unique_ptr( + webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, mangled_sdp)), + rtc::scoped_refptr(this)); - mPeerConnection->SetLocalDescription(std::unique_ptr(webrtc::CreateSessionDescription(webrtc::SdpType::kOffer, sdp_mangled_stream.str())), - rtc::scoped_refptr(this)); } void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) @@ -1135,6 +1135,14 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() break; case webrtc::DataChannelInterface::kClosed: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; + // if the data channel is up, we need to shut it down, holding off + // on termination of the peer connection until it's been closed. + if (mClosing) + { + // a close was requested, and the data channel has closed, + // so go ahead and call shutdownConnection again to clean up. + shutdownConnection(); + } break; default: break; diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index dab7774499..be2e5cdf68 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -192,9 +192,6 @@ class LLWebRTCSignalingObserver // Called when the data channel has been established and data // transfer can begin. virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; - - // Called when a peer connection has finished shutting down. - virtual void OnPeerConnectionShutdown() = 0; }; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 1f696e8c66..38810a29b5 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -346,21 +346,23 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, LLWebRTCImpl * mWebRTCImpl; + bool mClosing; + rtc::scoped_refptr mPeerConnectionFactory; - bool mMute; + bool mMute; // signaling - std::vector mSignalingObserverList; + std::vector mSignalingObserverList; std::vector> mCachedIceCandidates; - bool mAnswerReceived; + bool mAnswerReceived; - rtc::scoped_refptr mPeerConnection; - rtc::scoped_refptr mLocalStream; + rtc::scoped_refptr mPeerConnection; + rtc::scoped_refptr mLocalStream; // data - std::vector mDataObserverList; - rtc::scoped_refptr mDataChannel; + std::vector mDataObserverList; + rtc::scoped_refptr mDataChannel; }; } diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 1441e4e305..6aab67ca63 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2043,38 +2043,31 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() // callback from llwebrtc void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state) { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnIceGatheringStateImpl(state); }); -} + LL::WorkQueue::postMaybe(mMainQueue, + [=] { + LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL; -void LLVoiceWebRTCConnection::OnIceGatheringStateImpl(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state) -{ - LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL; - - switch (state) - { - case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE: - { - mIceCompleted = true; - break; - } - case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW: - { - mIceCompleted = false; - } - default: - break; - } + switch (state) + { + case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE: + { + mIceCompleted = true; + break; + } + case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW: + { + mIceCompleted = false; + } + default: + break; + } + }); } // callback from llwebrtc void LLVoiceWebRTCConnection::OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate& candidate) { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnIceCandidateImpl(candidate); }); -} - -void LLVoiceWebRTCConnection::OnIceCandidateImpl(const llwebrtc::LLWebRTCIceCandidate &candidate) -{ - mIceCandidates.push_back(candidate); + LL::WorkQueue::postMaybe(mMainQueue, [=] { mIceCandidates.push_back(candidate); }); } void LLVoiceWebRTCConnection::onIceUpdateComplete(bool ice_completed, const LLSD &result) @@ -2205,37 +2198,31 @@ void LLVoiceWebRTCConnection::processIceUpdates() // and is passed to the simulator via a CAP, which then passes // it on to the Secondlife WebRTC server. -// callback from llwebrtc -void LLVoiceWebRTCConnection::OnOfferAvailable(const std::string &sdp) -{ - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnOfferAvailableImpl(sdp); }); -} - // // The LLWebRTCVoiceConnection object will not be deleted // before the webrtc connection itself is shut down, so // we shouldn't be getting this callback on a nonexistant // this pointer. -void LLVoiceWebRTCConnection::OnOfferAvailableImpl(const std::string &sdp) -{ - if (mShutDown) - { - return; - } - LL_DEBUGS("Voice") << "On Offer Available." << LL_ENDL; - mChannelSDP = sdp; - if (mVoiceConnectionState == VOICE_STATE_WAIT_FOR_SESSION_START) - { - mVoiceConnectionState = VOICE_STATE_REQUEST_CONNECTION; - } -} // callback from llwebrtc -void LLVoiceWebRTCConnection::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface* audio_interface) +void LLVoiceWebRTCConnection::OnOfferAvailable(const std::string &sdp) { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnAudioEstablishedImpl(audio_interface); }); + LL::WorkQueue::postMaybe(mMainQueue, + [=] { + if (mShutDown) + { + return; + } + LL_DEBUGS("Voice") << "On Offer Available." << LL_ENDL; + mChannelSDP = sdp; + if (mVoiceConnectionState == VOICE_STATE_WAIT_FOR_SESSION_START) + { + mVoiceConnectionState = VOICE_STATE_REQUEST_CONNECTION; + } + }); } + // // The LLWebRTCVoiceConnection object will not be deleted // before the webrtc connection itself is shut down, so @@ -2243,48 +2230,42 @@ void LLVoiceWebRTCConnection::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterfac // this pointer. // nor should audio_interface be invalid if the LLWebRTCVoiceConnection // is shut down. -void LLVoiceWebRTCConnection::OnAudioEstablishedImpl(llwebrtc::LLWebRTCAudioInterface *audio_interface) -{ - if (mShutDown) - { - return; - } - LL_DEBUGS("Voice") << "On AudioEstablished." << LL_ENDL; - mWebRTCAudioInterface = audio_interface; - setVoiceConnectionState(VOICE_STATE_SESSION_ESTABLISHED); -} // callback from llwebrtc -void LLVoiceWebRTCConnection::OnRenegotiationNeeded() +void LLVoiceWebRTCConnection::OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface* audio_interface) { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnRenegotiationNeededImpl(); }); + LL::WorkQueue::postMaybe(mMainQueue, + [=] { + if (mShutDown) + { + return; + } + LL_DEBUGS("Voice") << "On AudioEstablished." << LL_ENDL; + mWebRTCAudioInterface = audio_interface; + setVoiceConnectionState(VOICE_STATE_SESSION_ESTABLISHED); + }); } + // // The LLWebRTCVoiceConnection object will not be deleted // before the webrtc connection itself is shut down, so // we shouldn't be getting this callback on a nonexistant // this pointer. -void LLVoiceWebRTCConnection::OnRenegotiationNeededImpl() -{ - LL_DEBUGS("Voice") << "Voice channel requires renegotiation." << LL_ENDL; - if (!mShutDown) - { - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); - } -} // callback from llwebrtc -void LLVoiceWebRTCConnection::OnPeerConnectionShutdown() +void LLVoiceWebRTCConnection::OnRenegotiationNeeded() { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnPeerConnectionShutdownImpl(); }); + LL::WorkQueue::postMaybe(mMainQueue, + [=] { + LL_DEBUGS("Voice") << "Voice channel requires renegotiation." << LL_ENDL; + if (!mShutDown) + { + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + } + }); } -void LLVoiceWebRTCConnection::OnPeerConnectionShutdownImpl() -{ - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); - mOutstandingRequests--; // shut down is an async call which is handled on a webrtc thread. -} void LLVoiceWebRTCConnection::setMuteMic(bool muted) { @@ -2354,7 +2335,7 @@ void LLVoiceWebRTCConnection::sendData(const std::string &data) // Tell the simulator that we're shutting down a voice connection. // The simulator will pass this on to the Secondlife WebRTC server. -bool LLVoiceWebRTCConnection::breakVoiceConnection(bool corowait) +void LLVoiceWebRTCConnection::breakVoiceConnection() { LL_DEBUGS("Voice") << "Disconnecting voice." << LL_ENDL; if (mWebRTCDataInterface) @@ -2367,96 +2348,53 @@ bool LLVoiceWebRTCConnection::breakVoiceConnection(bool corowait) if (!regionp || !regionp->capabilitiesReceived()) { LL_DEBUGS("Voice") << "no capabilities for voice provisioning; waiting " << LL_ENDL; - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); - return false; + return; } std::string url = regionp->getCapability("ProvisionVoiceAccountRequest"); if (url.empty()) { - return false; + return; } LL_DEBUGS("Voice") << "region ready for voice break; url=" << url << LL_ENDL; - LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); - LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("parcelVoiceInfoRequest", httpPolicy)); - LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); - LLVoiceWebRTCStats::getInstance()->provisionAttemptStart(); LLSD body; body["logout"] = TRUE; body["viewer_session"] = mViewerSession; body["voice_server_type"] = REPORTED_VOICE_SERVER_TYPE; - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestSuccess, this, _1), - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestFailure, this, url, 3, body, _1)); - setVoiceConnectionState(VOICE_STATE_WAIT_FOR_EXIT); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( + new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", + LLCore::HttpRequest::DEFAULT_POLICY_ID)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + + httpOpts->setWantHeaders(true); + mOutstandingRequests++; - return true; -} + setVoiceConnectionState(VOICE_STATE_WAIT_FOR_EXIT); -void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess(const LLSD &result) -{ - mOutstandingRequests--; - if (LLWebRTCVoiceClient::isShuttingDown()) - { - return; - } + // tell the server to shut down the connection as a courtesy. + // shutdownConnection will drop the WebRTC connection which will + // also shut things down. + LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); if (mWebRTCPeerConnectionInterface) { - if (mWebRTCPeerConnectionInterface->shutdownConnection()) - { - mOutstandingRequests++; - } - } - else - { - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); - } -} - -void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure(std::string url, int retries, LLSD body, const LLSD &result) -{ - if (LLWebRTCVoiceClient::isShuttingDown()) - { - mOutstandingRequests--; - return; - } - if (retries >= 0) - { - // retry a few times. - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestSuccess, this, _1), - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestFailure, this, url, retries - 1, body, _1)); - return; - } - if (mWebRTCPeerConnectionInterface) - { - mOutstandingRequests++; mWebRTCPeerConnectionInterface->shutdownConnection(); } - else - { - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); - } + setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); + mOutstandingRequests--; } - // Tell the simulator to tell the Secondlife WebRTC server that we want a voice // connection. The SDP is sent up as part of this, and the simulator will respond // with an 'answer' which is in the form of another SDP. The webrtc library // will use the offer and answer to negotiate the session. -bool LLVoiceWebRTCSpatialConnection::requestVoiceConnection() +void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() { LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); @@ -2464,13 +2402,15 @@ bool LLVoiceWebRTCSpatialConnection::requestVoiceConnection() if (!regionp || !regionp->capabilitiesReceived()) { LL_DEBUGS("Voice") << "no capabilities for voice provisioning; waiting " << LL_ENDL; - return false; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + return; } std::string url = regionp->getCapability("ProvisionVoiceAccountRequest"); if (url.empty()) { - return false; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + return; } LL_DEBUGS("Voice") << "region ready for voice provisioning; url=" << url << LL_ENDL; @@ -2489,27 +2429,42 @@ bool LLVoiceWebRTCSpatialConnection::requestVoiceConnection() } body["channel_type"] = "local"; body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( + new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::requestVoiceConnection", + LLCore::HttpRequest::DEFAULT_POLICY_ID)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceConnectionRequestSuccess, this, _1), - boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceConnectionRequestFailure, this, url, 3, body, _1)); + httpOpts->setWantHeaders(true); mOutstandingRequests++; - return true; + LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); + + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + if (!status) + { + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + } + else + { + OnVoiceConnectionRequestSuccess(result); + } + mOutstandingRequests--; } void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result) { - mOutstandingRequests--; if (LLWebRTCVoiceClient::isShuttingDown()) { return; } LLVoiceWebRTCStats::getInstance()->provisionAttemptEnd(true); - if (result.has("viewer_session") && result.has("jsep") && result["jsep"].has("type") && result["jsep"]["type"] == "answer" && + if (result.has("viewer_session") && + result.has("jsep") && + result["jsep"].has("type") && + result["jsep"]["type"] == "answer" && result["jsep"].has("sdp")) { mRemoteChannelSDP = result["jsep"]["sdp"].asString(); @@ -2524,35 +2479,9 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result LL_DEBUGS("Voice") << "ProvisionVoiceAccountRequest response" << " channel sdp " << mRemoteChannelSDP << LL_ENDL; - mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP); } - -void LLVoiceWebRTCConnection::OnVoiceConnectionRequestFailure(std::string url, int retries, LLSD body, const LLSD &result) -{ - if (LLWebRTCVoiceClient::isShuttingDown()) - { - mOutstandingRequests--; - return; - } - if (retries >= 0) - { - LL_WARNS("Voice") << "Failure connecting to voice, retrying." << body << " RESULT: " << result << LL_ENDL; - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess, this, _1), - boost::bind(&LLVoiceWebRTCConnection::OnVoiceConnectionRequestFailure, this, url, retries - 1, body, _1)); - return; - } - LL_WARNS("Voice") << "Unable to connect voice." << body << " RESULT: " << result << LL_ENDL; - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); - mOutstandingRequests--; -} - - // Primary state machine for negotiating a single voice connection to the // Secondlife WebRTC server. bool LLVoiceWebRTCConnection::connectionStateMachine() @@ -2599,14 +2528,9 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() // Ask the sim to ask the Secondlife WebRTC server for a connection to // a given voice channel. On completion, we'll move on to the // VOICE_STATE_SESSION_ESTABLISHED via a callback on a webrtc thread. - if (!requestVoiceConnection()) - { - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); - } - else - { - setVoiceConnectionState(VOICE_STATE_CONNECTION_WAIT); - } + setVoiceConnectionState(VOICE_STATE_CONNECTION_WAIT); + LLCoros::getInstance()->launch("LLVoiceWebRTCConnection::requestVoiceConnectionCoro", + boost::bind(&LLVoiceWebRTCConnection::requestVoiceConnectionCoro, this)); break; case VOICE_STATE_CONNECTION_WAIT: @@ -2680,7 +2604,8 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() break; case VOICE_STATE_DISCONNECT: - breakVoiceConnection(true); + LLCoros::instance().launch("LLVoiceWebRTCConnection::breakVoiceConnection", + boost::bind(&LLVoiceWebRTCConnection::breakVoiceConnection, this)); break; case VOICE_STATE_WAIT_FOR_EXIT: @@ -2732,7 +2657,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() // llwebrtc callback void LLVoiceWebRTCConnection::OnDataReceived(const std::string& data, bool binary) { - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnDataReceivedImpl(data, binary); }); + LL::WorkQueue::postMaybe(mMainQueue, [=] { LLVoiceWebRTCConnection::OnDataReceivedImpl(data, binary); }); } // @@ -2853,12 +2778,6 @@ void LLVoiceWebRTCConnection::OnDataReceivedImpl(const std::string &data, bool b } } -// llwebrtc callback -void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) -{ - LL::WorkQueue::postMaybe(mMainQueue, [=] { OnDataChannelReadyImpl(data_interface); }); -} - // // The LLWebRTCVoiceConnection object will not be deleted // before the webrtc connection itself is shut down, so @@ -2866,18 +2785,23 @@ void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface // this pointer. // nor should data_interface be invalid if the LLWebRTCVoiceConnection // is shut down. -void LLVoiceWebRTCConnection::OnDataChannelReadyImpl(llwebrtc::LLWebRTCDataInterface *data_interface) -{ - if (mShutDown) - { - return; - } - if (data_interface) - { - mWebRTCDataInterface = data_interface; - mWebRTCDataInterface->setDataObserver(this); - } +// llwebrtc callback +void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface *data_interface) +{ + LL::WorkQueue::postMaybe(mMainQueue, + [=] { + if (mShutDown) + { + return; + } + + if (data_interface) + { + mWebRTCDataInterface = data_interface; + mWebRTCDataInterface->setDataObserver(this); + } + }); } // tell the Secondlife WebRTC server that @@ -2974,7 +2898,7 @@ LLVoiceWebRTCAdHocConnection::~LLVoiceWebRTCAdHocConnection() // as they go to a different set of Secondlife WebRTC servers. // They also require credentials for the given channels. // So, we have a separate requestVoiceConnection call. -bool LLVoiceWebRTCAdHocConnection::requestVoiceConnection() +void LLVoiceWebRTCAdHocConnection::requestVoiceConnection() { LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); @@ -2982,13 +2906,15 @@ bool LLVoiceWebRTCAdHocConnection::requestVoiceConnection() if (!regionp || !regionp->capabilitiesReceived()) { LL_DEBUGS("Voice") << "no capabilities for voice provisioning; waiting " << LL_ENDL; - return false; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + return; } std::string url = regionp->getCapability("ProvisionVoiceAccountRequest"); if (url.empty()) { - return false; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + return; } LLVoiceWebRTCStats::getInstance()->provisionAttemptStart(); @@ -3004,12 +2930,26 @@ bool LLVoiceWebRTCAdHocConnection::requestVoiceConnection() body["channel_type"] = "multiagent"; body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCAdHocConnection::OnVoiceConnectionRequestSuccess, this, _1), - boost::bind(&LLVoiceWebRTCAdHocConnection::OnVoiceConnectionRequestFailure, this, url, 3, body, _1)); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( + new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::requestVoiceConnection", + LLCore::HttpRequest::DEFAULT_POLICY_ID)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + + httpOpts->setWantHeaders(true); mOutstandingRequests++; - return true; + LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); + + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + if (!status) + { + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + } + else + { + OnVoiceConnectionRequestSuccess(result); + } + mOutstandingRequests--; } diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index b0c214dfd5..21fc79420b 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -588,16 +588,8 @@ class LLVoiceWebRTCConnection : void OnOfferAvailable(const std::string &sdp) override; void OnRenegotiationNeeded() override; void OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) override; - void OnPeerConnectionShutdown() override; //@} - void OnIceGatheringStateImpl(EIceGatheringState state); - void OnIceCandidateImpl(const llwebrtc::LLWebRTCIceCandidate &candidate); - void OnOfferAvailableImpl(const std::string &sdp); - void OnRenegotiationNeededImpl(); - void OnAudioEstablishedImpl(llwebrtc::LLWebRTCAudioInterface *audio_interface); - void OnPeerConnectionShutdownImpl(); - ///////////////////////// /// @name Data Notification /// LLWebRTCDataObserver @@ -607,7 +599,6 @@ class LLVoiceWebRTCConnection : //@} void OnDataReceivedImpl(const std::string &data, bool binary); - void OnDataChannelReadyImpl(llwebrtc::LLWebRTCDataInterface *data_interface); void sendJoin(); void sendData(const std::string &data); @@ -635,7 +626,6 @@ class LLVoiceWebRTCConnection : } void OnVoiceConnectionRequestSuccess(const LLSD &body); - void OnVoiceConnectionRequestFailure(std::string url, int retries, LLSD body, const LLSD &result); protected: typedef enum e_voice_connection_state @@ -680,11 +670,10 @@ class LLVoiceWebRTCConnection : return mVoiceConnectionState; } - virtual bool requestVoiceConnection() = 0; + virtual void requestVoiceConnection() = 0; + void requestVoiceConnectionCoro() { requestVoiceConnection(); } - bool breakVoiceConnection(bool wait); - void OnVoiceDisconnectionRequestSuccess(const LLSD &body); - void OnVoiceDisconnectionRequestFailure(std::string url, int retries, LLSD body, const LLSD &result); + void breakVoiceConnection(); LLUUID mRegionID; LLUUID mViewerSession; @@ -731,7 +720,7 @@ class LLVoiceWebRTCSpatialConnection : protected: - bool requestVoiceConnection() override; + void requestVoiceConnection() override; S32 mParcelLocalID; }; @@ -746,7 +735,7 @@ class LLVoiceWebRTCAdHocConnection : public LLVoiceWebRTCConnection bool isSpatial() override { return false; } protected: - bool requestVoiceConnection() override; + void requestVoiceConnection() override; std::string mCredentials; }; From 0e2fc4d35ea1fdef0028a5132602985da2e1c463 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 19 Mar 2024 17:05:56 -0700 Subject: [PATCH 03/21] Clear out observers when terminating a connection --- autobuild.xml | 24 +++++++++++++++++++----- indra/llwebrtc/llwebrtc.cpp | 8 -------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/autobuild.xml b/autobuild.xml index 3a3e2e5f14..f9c99514a1 100644 --- a/autobuild.xml +++ b/autobuild.xml @@ -2901,15 +2901,29 @@ Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors creds github hash - 8b0191fae0860782a3e79b886364129c433cfd6b + a49fb3bb8aaf8325e7c6c4b6036db3da16afa2c9 hash_algorithm sha1 url - https://api.github.com/repos/secondlife/3p-webrtc-build/releases/assets/157215889 + https://github.com/secondlife/3p-webrtc-build/releases/download/m114.5735.08.53/webrtc-m114.5735.08.53.8337236647-darwin64-8337236647.tar.zst name darwin64 + linux64 + + archive + + hash + 598baa054f63624a8e16883541c1f3dc7aa15a8a + hash_algorithm + sha1 + url + https://github.com/secondlife/3p-webrtc-build/releases/download/m114.5735.08.53/webrtc-m114.5735.08.53.8337236647-linux64-8337236647.tar.zst + + name + linux64 + windows64 archive @@ -2917,11 +2931,11 @@ Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors creds github hash - 64eccac933cee532dc065d9f9729a21d8347aae4 + 59d5f2e40612ab7b0b1a5da8ba288f48d5979216 hash_algorithm sha1 url - https://api.github.com/repos/secondlife/3p-webrtc-build/releases/assets/157215892 + https://github.com/secondlife/3p-webrtc-build/releases/download/m114.5735.08.53/webrtc-m114.5735.08.53.8337236647-windows64-8337236647.tar.zst name windows64 @@ -2934,7 +2948,7 @@ Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors vcs_url https://github.com/secondlife/3p-webrtc-build version - m114.5735.08.52.8319849783 + m114.5735.08.53.8337236647 xmlrpc-epi diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 283b94dd3c..34d950b804 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1135,14 +1135,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange() break; case webrtc::DataChannelInterface::kClosed: RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State closed"; - // if the data channel is up, we need to shut it down, holding off - // on termination of the peer connection until it's been closed. - if (mClosing) - { - // a close was requested, and the data channel has closed, - // so go ahead and call shutdownConnection again to clean up. - shutdownConnection(); - } break; default: break; From 4b709e7944ad34bf82c63b8ee57661598bc2292f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Wed, 20 Mar 2024 11:15:12 -0700 Subject: [PATCH 04/21] Turn procesIceUpdates into a coroutine to chop up the work a bit. --- indra/llwebrtc/llwebrtc.cpp | 3 + indra/newview/llvoicewebrtc.cpp | 165 +++++++++++++------------------- indra/newview/llvoicewebrtc.h | 7 +- 3 files changed, 71 insertions(+), 104 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 34d950b804..1f129cf514 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -589,6 +589,9 @@ void LLWebRTCPeerConnectionImpl::terminate() rtc::scoped_refptr localStream; mLocalStream.swap(localStream); + mSignalingObserverList.clear(); + mDataObserverList.clear(); + mWebRTCImpl->PostSignalingTask( [=]() { diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 6aab67ca63..3446cf38f6 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -1997,7 +1997,6 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID ®ionID, const s mVoiceConnectionState(VOICE_STATE_START_SESSION), mMuted(true), mShutDown(false), - mTrickling(false), mIceCompleted(false), mSpeakerVolume(0.0), mMicGain(0.0), @@ -2070,123 +2069,90 @@ void LLVoiceWebRTCConnection::OnIceCandidate(const llwebrtc::LLWebRTCIceCandidat LL::WorkQueue::postMaybe(mMainQueue, [=] { mIceCandidates.push_back(candidate); }); } -void LLVoiceWebRTCConnection::onIceUpdateComplete(bool ice_completed, const LLSD &result) +void LLVoiceWebRTCConnection::processIceUpdates() { - mOutstandingRequests--; - if (LLWebRTCVoiceClient::isShuttingDown()) - { - return; - } - mTrickling = false; + mOutstandingRequests++; + LLCoros::getInstance()->launch("LLVoiceWebRTCConnection::requestVoiceConnectionCoro", + boost::bind(&LLVoiceWebRTCConnection::processIceUpdatesCoro, this)); } -void LLVoiceWebRTCConnection::onIceUpdateError(int retries, std::string url, LLSD body, bool ice_completed, const LLSD &result) -{ - if (LLWebRTCVoiceClient::isShuttingDown()) - { - mOutstandingRequests--; - return; - } - LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); - LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("voiceAccountProvision", httpPolicy)); - - if (retries >= 0) - { - LL_WARNS("Voice") << "Unable to complete ice trickling voice account, retrying. " << result << LL_ENDL; - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCConnection::onIceUpdateComplete, this, ice_completed, _1), - boost::bind(&LLVoiceWebRTCConnection::onIceUpdateError, this, retries - 1, url, body, ice_completed, _1)); - return; - } - - LL_WARNS("Voice") << "Unable to complete ice trickling voice account, restarting connection. " << result << LL_ENDL; - if (!mShutDown) - { - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); - } - mTrickling = false; - - mOutstandingRequests--; -} - - // Ice candidates may be streamed in before or after the SDP offer is available (see below) // This function determines whether candidates are available to send to the Secondlife WebRTC // server via the simulator. If so, and there are no more candidates, this code // will make the cap call to the server sending up the ICE candidates. -void LLVoiceWebRTCConnection::processIceUpdates() +void LLVoiceWebRTCConnection::processIceUpdatesCoro() { if (mShutDown || LLWebRTCVoiceClient::isShuttingDown()) { + mOutstandingRequests--; return; } bool iceCompleted = false; LLSD body; + if (!mIceCandidates.empty() || mIceCompleted) { - if (!mTrickling) + LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); + if (!regionp || !regionp->capabilitiesReceived()) { - if (!mIceCandidates.empty() || mIceCompleted) + LL_DEBUGS("Voice") << "no capabilities for ice gathering; waiting " << LL_ENDL; + mOutstandingRequests--; + return; + } + + std::string url = regionp->getCapability("VoiceSignalingRequest"); + if (url.empty()) + { + mOutstandingRequests--; + return; + } + + LL_DEBUGS("Voice") << "region ready to complete voice signaling; url=" << url << LL_ENDL; + if (!mIceCandidates.empty()) + { + LLSD candidates = LLSD::emptyArray(); + for (auto &ice_candidate : mIceCandidates) { - LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); - if (!regionp || !regionp->capabilitiesReceived()) - { - LL_DEBUGS("Voice") << "no capabilities for ice gathering; waiting " << LL_ENDL; - return; - } - - std::string url = regionp->getCapability("VoiceSignalingRequest"); - if (url.empty()) - { - return; - } - - LL_DEBUGS("Voice") << "region ready to complete voice signaling; url=" << url << LL_ENDL; - if (!mIceCandidates.empty()) - { - LLSD candidates = LLSD::emptyArray(); - for (auto &ice_candidate : mIceCandidates) - { - LLSD body_candidate; - body_candidate["sdpMid"] = ice_candidate.mSdpMid; - body_candidate["sdpMLineIndex"] = ice_candidate.mMLineIndex; - body_candidate["candidate"] = ice_candidate.mCandidate; - candidates.append(body_candidate); - } - body["candidates"] = candidates; - mIceCandidates.clear(); - } - else if (mIceCompleted) - { - LLSD body_candidate; - body_candidate["completed"] = true; - body["candidate"] = body_candidate; - iceCompleted = mIceCompleted; - mIceCompleted = false; - } - - body["viewer_session"] = mViewerSession; - body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; - - LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); - LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( - new LLCoreHttpUtil::HttpCoroutineAdapter("voiceAccountProvision", httpPolicy)); - LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); - LLCore::HttpOptions::ptr_t httpOpts = LLCore::HttpOptions::ptr_t(new LLCore::HttpOptions); - LLCoreHttpUtil::HttpCoroutineAdapter::callbackHttpPost( - url, - LLCore::HttpRequest::DEFAULT_POLICY_ID, - body, - boost::bind(&LLVoiceWebRTCSpatialConnection::onIceUpdateComplete, this, iceCompleted, _1), - boost::bind(&LLVoiceWebRTCSpatialConnection::onIceUpdateError, this, 3, url, body, iceCompleted, _1)); - mOutstandingRequests++; - mTrickling = true; + LLSD body_candidate; + body_candidate["sdpMid"] = ice_candidate.mSdpMid; + body_candidate["sdpMLineIndex"] = ice_candidate.mMLineIndex; + body_candidate["candidate"] = ice_candidate.mCandidate; + candidates.append(body_candidate); } + body["candidates"] = candidates; + mIceCandidates.clear(); + } + else if (mIceCompleted) + { + LLSD body_candidate; + body_candidate["completed"] = true; + body["candidate"] = body_candidate; + iceCompleted = mIceCompleted; + mIceCompleted = false; + } + + body["viewer_session"] = mViewerSession; + body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; + + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( + new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", + LLCore::HttpRequest::DEFAULT_POLICY_ID)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + + httpOpts->setWantHeaders(true); + + LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + if (!status) + { + // couldn't trickle the candidates, so restart the session. + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); } } + mOutstandingRequests--; } @@ -2439,8 +2405,8 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() mOutstandingRequests++; LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); - LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; - LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); if (!status) { @@ -2497,7 +2463,6 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() setVoiceConnectionState(VOICE_STATE_DISCONNECT); break; } - mTrickling = false; mIceCompleted = false; setVoiceConnectionState(VOICE_STATE_WAIT_FOR_SESSION_START); // tell the webrtc library that we want a connection. The library will diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index 21fc79420b..4c82f5c9fa 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -603,10 +603,10 @@ class LLVoiceWebRTCConnection : void sendJoin(); void sendData(const std::string &data); - virtual void processIceUpdates(); - virtual void onIceUpdateComplete(bool ice_completed, const LLSD &result); - virtual void onIceUpdateError(int retries, std::string url, LLSD body, bool ice_completed, const LLSD &result); + void processIceUpdates(); + void processIceUpdatesCoro(); + virtual void setMuteMic(bool muted); virtual void setMicGain(F32 volume); virtual void setSpeakerVolume(F32 volume); @@ -697,7 +697,6 @@ class LLVoiceWebRTCConnection : std::vector mIceCandidates; bool mIceCompleted; - bool mTrickling; llwebrtc::LLWebRTCPeerConnectionInterface *mWebRTCPeerConnectionInterface; llwebrtc::LLWebRTCAudioInterface *mWebRTCAudioInterface; From e242c129f9d56161496a50397b79da1e444e8de4 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Mar 2024 16:21:02 -0700 Subject: [PATCH 05/21] Add Tracy categories for WebRTC Voice Also: * Fix a few crashes. * Only send position data when it changes. --- indra/llcommon/llprofilercategories.h | 13 +++++- indra/llwebrtc/llwebrtc.cpp | 16 ++++++- indra/newview/llvoicewebrtc.cpp | 63 ++++++++++++++++++++++++--- indra/newview/llvoicewebrtc.h | 2 +- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/indra/llcommon/llprofilercategories.h b/indra/llcommon/llprofilercategories.h index 617431f629..0de343d020 100644 --- a/indra/llcommon/llprofilercategories.h +++ b/indra/llcommon/llprofilercategories.h @@ -1,5 +1,5 @@ /** - * @file llprofiler_ategories.h + * @file llprofilercategories.h * @brief Profiling categories to minimize Tracy memory usage when viewing captures. * * $LicenseInfo:firstyear=2022&license=viewerlgpl$ @@ -33,7 +33,7 @@ // LL_PROFILER_CATEGORY_ENABLE_DRAWPOOL // LL_PROFILER_CATEGORY_ENABLE_LLSD // LL_PROFILER_CATEGORY_ENABLE_MEMORY -// LL_PROFILER_CATEGORY_ENABLE_SHADERS +// LL_PROFILER_CATEGORY_ENABLE_SHADER // // NOTE: You can still manually use: // LL_PROFILE_ZONE_SCOPED(); @@ -67,6 +67,7 @@ #define LL_PROFILER_CATEGORY_ENABLE_VERTEX 1 #define LL_PROFILER_CATEGORY_ENABLE_VOLUME 1 #define LL_PROFILER_CATEGORY_ENABLE_WIN32 1 +#define LL_PROFILER_CATEGORY_ENABLE_VOICE 1 #if LL_PROFILER_CATEGORY_ENABLE_APP #define LL_PROFILE_ZONE_NAMED_CATEGORY_APP LL_PROFILE_ZONE_NAMED @@ -276,5 +277,13 @@ #define LL_PROFILE_ZONE_SCOPED_CATEGORY_WIN32 #endif +#if LL_PROFILER_CATEGORY_ENABLE_VOICE +#define LL_PROFILE_ZONE_NAMED_CATEGORY_VOICE LL_PROFILE_ZONE_NAMED +#define LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE LL_PROFILE_ZONE_SCOPED +#else +#define LL_PROFILE_ZONE_NAMED_CATEGORY_VOICE(name) +#define LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE +#endif + #endif // LL_PROFILER_CATEGORIES_H diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 1f129cf514..a92b480e3a 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1103,7 +1103,10 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError } } mCachedIceCandidates.clear(); - OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + if (mPeerConnection) + { + OnIceGatheringChange(mPeerConnection->ice_gathering_state()); + } } @@ -1120,6 +1123,10 @@ void LLWebRTCPeerConnectionImpl::OnSetLocalDescriptionComplete(webrtc::RTCError void LLWebRTCPeerConnectionImpl::OnStateChange() { + if (!mDataChannel) + { + return; + } RTC_LOG(LS_INFO) << __FUNCTION__ << " Data Channel State: " << webrtc::DataChannelInterface::DataStateString(mDataChannel->state()); switch (mDataChannel->state()) { @@ -1163,7 +1170,12 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary) { rtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length()); webrtc::DataBuffer buffer(cowBuffer, binary); - mDataChannel->Send(buffer); + mWebRTCImpl->PostNetworkTask([this, buffer]() { + if (mDataChannel) + { + mDataChannel->Send(buffer); + } + }); } } diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 3446cf38f6..57c95777c2 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -290,6 +290,8 @@ const LLVoiceVersionInfo& LLWebRTCVoiceClient::getVersion() void LLWebRTCVoiceClient::updateSettings() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + setVoiceEnabled(LLVoiceClient::getInstance()->voiceEnabled()); setEarLocation(gSavedSettings.getS32("VoiceEarLocation")); @@ -314,6 +316,7 @@ void LLWebRTCVoiceClient::removeObserver(LLVoiceClientParticipantObserver *obser void LLWebRTCVoiceClient::notifyParticipantObservers() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE for (observer_set_t::iterator it = mParticipantObservers.begin(); it != mParticipantObservers.end();) { LLVoiceClientParticipantObserver *observer = *it; @@ -335,6 +338,8 @@ void LLWebRTCVoiceClient::removeObserver(LLVoiceClientStatusObserver *observer) void LLWebRTCVoiceClient::notifyStatusObservers(LLVoiceClientStatusObserver::EStatusType status) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LL_DEBUGS("Voice") << "( " << LLVoiceClientStatusObserver::status2string(status) << " )" << " mSession=" << mSession << LL_ENDL; @@ -401,6 +406,7 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() LLMuteList::getInstance()->addObserver(this); while (!sShuttingDown) { + LL_PROFILE_ZONE_NAMED_CATEGORY_VOICE("voiceConnectionCoroLoop") // TODO: Doing some measurement and calculation here, // we could reduce the timeout to take into account the // time spent on the previous loop to have the loop @@ -487,7 +493,7 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() sessionState::processSessionStates(); if (mProcessChannels && voiceEnabled && !mHidden) { - sendPositionUpdate(true); + sendPositionUpdate(false); updateOwnVolume(); } } @@ -516,6 +522,8 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() // for cross-region voice. void LLWebRTCVoiceClient::updateNeighboringRegions() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + static const std::vector neighbors {LLVector3d(0.0f, 1.0f, 0.0f), LLVector3d(0.707f, 0.707f, 0.0f), LLVector3d(1.0f, 0.0f, 0.0f), LLVector3d(0.707f, -0.707f, 0.0f), LLVector3d(0.0f, -1.0f, 0.0f), LLVector3d(-0.707f, -0.707f, 0.0f), @@ -546,6 +554,8 @@ void LLWebRTCVoiceClient::updateNeighboringRegions() // shut down the current audio session to make room for the next one. void LLWebRTCVoiceClient::leaveAudioSession() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if(mSession) { LL_DEBUGS("Voice") << "leaving session: " << mSession->mChannelID << LL_ENDL; @@ -599,6 +609,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi void LLWebRTCVoiceClient::OnDevicesChangedImpl(const llwebrtc::LLWebRTCVoiceDeviceList &render_devices, const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + std::string inputDevice = gSavedSettings.getString("VoiceInputAudioDevice"); std::string outputDevice = gSavedSettings.getString("VoiceOutputAudioDevice"); @@ -782,6 +794,8 @@ void LLWebRTCVoiceClient::setHidden(bool hidden) // notify the observers. void LLWebRTCVoiceClient::OnConnectionEstablished(const std::string &channelID, const LLUUID ®ionID) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (gAgent.getRegion()->getRegionID() == regionID) { if (mNextSession && mNextSession->mChannelID == channelID) @@ -854,6 +868,8 @@ void LLWebRTCVoiceClient::setEarLocation(S32 loc) void LLWebRTCVoiceClient::updatePosition(void) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LLViewerRegion *region = gAgent.getRegion(); if (region && isAgentAvatarValid()) { @@ -974,6 +990,8 @@ void LLWebRTCVoiceClient::enforceTether() // standard 50m void LLWebRTCVoiceClient::sendPositionUpdate(bool force) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + Json::FastWriter writer; std::string spatial_data; @@ -1085,6 +1103,8 @@ LLWebRTCVoiceClient::participantStatePtr_t LLWebRTCVoiceClient::addParticipantBy void LLWebRTCVoiceClient::removeParticipantByID(const std::string &channelID, const LLUUID &id) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + participantStatePtr_t result; LLWebRTCVoiceClient::sessionState::ptr_t session = sessionState::matchSessionByChannelID(channelID); if (session) @@ -1115,6 +1135,9 @@ LLWebRTCVoiceClient::participantState::participantState(const LLUUID& agent_id) LLWebRTCVoiceClient::participantStatePtr_t LLWebRTCVoiceClient::sessionState::addParticipant(const LLUUID& agent_id) { + + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + participantStatePtr_t result; participantUUIDMap::iterator iter = mParticipantsByUUID.find(agent_id); @@ -1150,6 +1173,8 @@ LLWebRTCVoiceClient::participantStatePtr_t LLWebRTCVoiceClient::sessionState::ad LLWebRTCVoiceClient::participantStatePtr_t LLWebRTCVoiceClient::sessionState::findParticipantByID(const LLUUID& id) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + participantStatePtr_t result; participantUUIDMap::iterator iter = mParticipantsByUUID.find(id); @@ -1163,6 +1188,8 @@ LLWebRTCVoiceClient::participantStatePtr_t LLWebRTCVoiceClient::sessionState::fi void LLWebRTCVoiceClient::sessionState::removeParticipant(const LLWebRTCVoiceClient::participantStatePtr_t &participant) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (participant) { participantUUIDMap::iterator iter = mParticipantsByUUID.find(participant->mAvatarID); @@ -1401,6 +1428,8 @@ void LLWebRTCVoiceClient::predSetMicGain(const LLWebRTCVoiceClient::sessionState void LLWebRTCVoiceClient::setVoiceEnabled(bool enabled) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LL_DEBUGS("Voice") << "( " << (enabled ? "enabled" : "disabled") << " )" << " was "<< (mVoiceEnabled ? "enabled" : "disabled") @@ -1803,6 +1832,8 @@ void LLWebRTCVoiceClient::sessionState::revive() void LLWebRTCVoiceClient::sessionState::processSessionStates() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + auto iter = mSessions.begin(); while (iter != mSessions.end()) { @@ -1822,6 +1853,8 @@ void LLWebRTCVoiceClient::sessionState::processSessionStates() // process the states on each connection associated with a session. bool LLWebRTCVoiceClient::sessionState::processConnectionStates() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + std::list::iterator iter = mWebRTCConnections.begin(); while (iter != mWebRTCConnections.end()) { @@ -1844,6 +1877,8 @@ bool LLWebRTCVoiceClient::sessionState::processConnectionStates() // on our location. bool LLWebRTCVoiceClient::estateSessionState::processConnectionStates() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (!mShuttingDown) { // Estate voice requires connection to neighboring regions. @@ -2082,6 +2117,8 @@ void LLVoiceWebRTCConnection::processIceUpdates() // will make the cap call to the server sending up the ICE candidates. void LLVoiceWebRTCConnection::processIceUpdatesCoro() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (mShutDown || LLWebRTCVoiceClient::isShuttingDown()) { mOutstandingRequests--; @@ -2135,7 +2172,7 @@ void LLVoiceWebRTCConnection::processIceUpdatesCoro() body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( - new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", + new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::processIceUpdatesCoro", LLCore::HttpRequest::DEFAULT_POLICY_ID)); LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); @@ -2301,8 +2338,10 @@ void LLVoiceWebRTCConnection::sendData(const std::string &data) // Tell the simulator that we're shutting down a voice connection. // The simulator will pass this on to the Secondlife WebRTC server. -void LLVoiceWebRTCConnection::breakVoiceConnection() +void LLVoiceWebRTCConnection::breakVoiceConnectionCoro() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LL_DEBUGS("Voice") << "Disconnecting voice." << LL_ENDL; if (mWebRTCDataInterface) { @@ -2362,6 +2401,8 @@ void LLVoiceWebRTCConnection::breakVoiceConnection() // will use the offer and answer to negotiate the session. void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); LL_DEBUGS("Voice") << "Requesting voice connection." << LL_ENDL; @@ -2421,6 +2462,8 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (LLWebRTCVoiceClient::isShuttingDown()) { return; @@ -2452,12 +2495,15 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result // Secondlife WebRTC server. bool LLVoiceWebRTCConnection::connectionStateMachine() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + processIceUpdates(); switch (getVoiceConnectionState()) { case VOICE_STATE_START_SESSION: { + LL_PROFILE_ZONE_NAMED_CATEGORY_VOICE("VOICE_STATE_START_SESSION") if (mShutDown) { setVoiceConnectionState(VOICE_STATE_DISCONNECT); @@ -2533,6 +2579,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() { sendJoin(); // tell the Secondlife WebRTC server that we're here via the data channel. setVoiceConnectionState(VOICE_STATE_SESSION_UP); + LLWebRTCVoiceClient::getInstance()->sendPositionUpdate(true); } break; } @@ -2569,8 +2616,8 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() break; case VOICE_STATE_DISCONNECT: - LLCoros::instance().launch("LLVoiceWebRTCConnection::breakVoiceConnection", - boost::bind(&LLVoiceWebRTCConnection::breakVoiceConnection, this)); + LLCoros::instance().launch("LLVoiceWebRTCConnection::breakVoiceConnectionCoro", + boost::bind(&LLVoiceWebRTCConnection::breakVoiceConnectionCoro, this)); break; case VOICE_STATE_WAIT_FOR_EXIT: @@ -2632,6 +2679,8 @@ void LLVoiceWebRTCConnection::OnDataReceived(const std::string& data, bool binar // this pointer. void LLVoiceWebRTCConnection::OnDataReceivedImpl(const std::string &data, bool binary) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + if (mShutDown) { return; @@ -2777,6 +2826,8 @@ void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface // to peers. void LLVoiceWebRTCConnection::sendJoin() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + Json::FastWriter writer; Json::Value root = Json::objectValue; Json::Value join_obj = Json::objectValue; @@ -2865,6 +2916,8 @@ LLVoiceWebRTCAdHocConnection::~LLVoiceWebRTCAdHocConnection() // So, we have a separate requestVoiceConnection call. void LLVoiceWebRTCAdHocConnection::requestVoiceConnection() { + LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE + LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); LL_DEBUGS("Voice") << "Requesting voice connection." << LL_ENDL; diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index 4c82f5c9fa..f6ddc6dbe7 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -673,7 +673,7 @@ class LLVoiceWebRTCConnection : virtual void requestVoiceConnection() = 0; void requestVoiceConnectionCoro() { requestVoiceConnection(); } - void breakVoiceConnection(); + void breakVoiceConnectionCoro(); LLUUID mRegionID; LLUUID mViewerSession; From 168081c7e9c4fb89209aae225f849573caaf905a Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Fri, 22 Mar 2024 17:00:34 -0700 Subject: [PATCH 06/21] Validate that we're not shutting down after coroutine processing Coroutine yields can end after shutdown is requested and voice connections are removed, so we need to check that a shutdown hasn't occured before attempting to touch connection objects. (CR issue) --- indra/newview/llvoicewebrtc.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 57c95777c2..80e7323b6f 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2180,6 +2180,12 @@ void LLVoiceWebRTCConnection::processIceUpdatesCoro() httpOpts->setWantHeaders(true); LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); + + if (LLWebRTCVoiceClient::isShuttingDown()) + { + return; + } + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); @@ -2386,6 +2392,11 @@ void LLVoiceWebRTCConnection::breakVoiceConnectionCoro() // also shut things down. LLSD result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); + if (LLWebRTCVoiceClient::isShuttingDown()) + { + return; + } + if (mWebRTCPeerConnectionInterface) { mWebRTCPeerConnectionInterface->shutdownConnection(); From 8d14df5984382de0aea12ba5d8ef4eba22b9976e Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 24 Mar 2024 20:42:32 -0700 Subject: [PATCH 07/21] Not sending proper voice server type down when breaking a connection --- indra/newview/llvoicewebrtc.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 6aab67ca63..a8bc468f4c 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2363,7 +2363,7 @@ void LLVoiceWebRTCConnection::breakVoiceConnection() LLSD body; body["logout"] = TRUE; body["viewer_session"] = mViewerSession; - body["voice_server_type"] = REPORTED_VOICE_SERVER_TYPE; + body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", @@ -2419,9 +2419,7 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() LLSD body; LLSD jsep; jsep["type"] = "offer"; - { - jsep["sdp"] = mChannelSDP; - } + jsep["sdp"] = mChannelSDP; body["jsep"] = jsep; if (mParcelLocalID != INVALID_PARCEL_ID) { From 92171a42c99a7fcd1751d2b6fae492c5aac466e0 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 24 Mar 2024 20:42:32 -0700 Subject: [PATCH 08/21] Not sending proper voice server type down when breaking a connection --- indra/newview/llvoicewebrtc.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 80e7323b6f..31ad5b7a5d 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2374,7 +2374,7 @@ void LLVoiceWebRTCConnection::breakVoiceConnectionCoro() LLSD body; body["logout"] = TRUE; body["viewer_session"] = mViewerSession; - body["voice_server_type"] = REPORTED_VOICE_SERVER_TYPE; + body["voice_server_type"] = WEBRTC_VOICE_SERVER_TYPE; LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter( new LLCoreHttpUtil::HttpCoroutineAdapter("LLVoiceWebRTCAdHocConnection::breakVoiceConnection", @@ -2437,9 +2437,7 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() LLSD body; LLSD jsep; jsep["type"] = "offer"; - { - jsep["sdp"] = mChannelSDP; - } + jsep["sdp"] = mChannelSDP; body["jsep"] = jsep; if (mParcelLocalID != INVALID_PARCEL_ID) { From e272e387d3ce2111d2a045f79671b3ebc3819267 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 24 Mar 2024 20:43:37 -0700 Subject: [PATCH 09/21] Throw 'area full' message when the voice server reports max users for voice --- indra/newview/llvoicewebrtc.cpp | 29 ++++++++++++++++++++++------- indra/newview/llvoicewebrtc.h | 6 +++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 31ad5b7a5d..72d343598b 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -837,18 +837,20 @@ void LLWebRTCVoiceClient::OnConnectionShutDown(const std::string &channelID, con } } } -void LLWebRTCVoiceClient::OnConnectionFailure(const std::string &channelID, const LLUUID ®ionID) +void LLWebRTCVoiceClient::OnConnectionFailure(const std::string &channelID, + const LLUUID ®ionID, + LLVoiceClientStatusObserver::EStatusType status_type) { LL_DEBUGS("Voice") << "A connection failed. channel:" << channelID << LL_ENDL; if (gAgent.getRegion()->getRegionID() == regionID) { if (mNextSession && mNextSession->mChannelID == channelID) { - LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(LLVoiceClientStatusObserver::ERROR_UNKNOWN); + LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(status_type); } else if (mSession && mSession->mChannelID == channelID) { - LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(LLVoiceClientStatusObserver::ERROR_UNKNOWN); + LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(status_type); } } } @@ -2030,6 +2032,7 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID ®ionID, const s mWebRTCAudioInterface(nullptr), mWebRTCDataInterface(nullptr), mVoiceConnectionState(VOICE_STATE_START_SESSION), + mCurrentStatus(LLVoiceClientStatusObserver::STATUS_VOICE_ENABLED), mMuted(true), mShutDown(false), mIceCompleted(false), @@ -2458,13 +2461,25 @@ void LLVoiceWebRTCSpatialConnection::requestVoiceConnection() LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); - if (!status) + if (status) { - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + OnVoiceConnectionRequestSuccess(result); } else { - OnVoiceConnectionRequestSuccess(result); + switch (status.getType()) + { + case HTTP_CONFLICT: + mCurrentStatus = LLVoiceClientStatusObserver::ERROR_CHANNEL_FULL; + break; + case HTTP_UNAUTHORIZED: + mCurrentStatus = LLVoiceClientStatusObserver::ERROR_CHANNEL_LOCKED; + break; + default: + mCurrentStatus = LLVoiceClientStatusObserver::ERROR_UNKNOWN; + break; + } + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); } mOutstandingRequests--; } @@ -2611,7 +2626,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() if (mRetryWaitPeriod++ * UPDATE_THROTTLE_SECONDS > mRetryWaitSecs) { // something went wrong, so notify that the connection has failed. - LLWebRTCVoiceClient::getInstance()->OnConnectionFailure(mChannelID, mRegionID); + LLWebRTCVoiceClient::getInstance()->OnConnectionFailure(mChannelID, mRegionID, mCurrentStatus); setVoiceConnectionState(VOICE_STATE_DISCONNECT); mRetryWaitPeriod = 0; if (mRetryWaitSecs < MAX_RETRY_WAIT_SECONDS) diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index f6ddc6dbe7..4fe0e756a7 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -203,7 +203,9 @@ public: void OnConnectionEstablished(const std::string& channelID, const LLUUID& regionID); void OnConnectionShutDown(const std::string &channelID, const LLUUID ®ionID); - void OnConnectionFailure(const std::string &channelID, const LLUUID& regionID); + void OnConnectionFailure(const std::string &channelID, + const LLUUID ®ionID, + LLVoiceClientStatusObserver::EStatusType status_type = LLVoiceClientStatusObserver::ERROR_UNKNOWN); void sendPositionUpdate(bool force); void updateOwnVolume(); @@ -675,6 +677,8 @@ class LLVoiceWebRTCConnection : void breakVoiceConnectionCoro(); + LLVoiceClientStatusObserver::EStatusType mCurrentStatus; + LLUUID mRegionID; LLUUID mViewerSession; std::string mChannelID; From cdae5ebc168d95a304b9905de7b66381723e402f Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 21:58:00 -0700 Subject: [PATCH 10/21] Add UI for managing echo cancellation, AGC, and noise control. Plumb audio settings through from webrtc to the sound preferences UI (still needs some tweaking, of course.) Also, choose stun servers based on grid. Ultimately, the stun stun servers will be passed up via login or something. --- indra/llwebrtc/llwebrtc.cpp | 98 ++++++++++++++----- indra/llwebrtc/llwebrtc.h | 44 ++++++++- indra/llwebrtc/llwebrtc_impl.h | 8 +- indra/newview/app_settings/settings.xml | 33 +++++++ indra/newview/llviewercontrol.cpp | 3 + indra/newview/llvoicevivox.cpp | 4 +- indra/newview/llvoicewebrtc.cpp | 35 ++++++- indra/newview/llvoicewebrtc.h | 2 +- .../xui/en/panel_preferences_sound.xml | 69 ++++++++++++- 9 files changed, 255 insertions(+), 41 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 34d950b804..c51bcfcdd5 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -157,7 +157,6 @@ LLWebRTCImpl::LLWebRTCImpl() : void LLWebRTCImpl::init() { - RTC_DCHECK(mPeerConnectionFactory); mPlayoutDevice = 0; mRecordingDevice = 0; rtc::InitializeSSL(); @@ -222,12 +221,10 @@ void LLWebRTCImpl::init() mPeerCustomProcessor = new LLCustomProcessor; webrtc::AudioProcessingBuilder apb; apb.SetCapturePostProcessing(std::unique_ptr(mPeerCustomProcessor)); - rtc::scoped_refptr apm = apb.Create(); + mAudioProcessingModule = apb.Create(); - // TODO: wire some of these to the primary interface and ultimately - // to the UI to allow user config. webrtc::AudioProcessing::Config apm_config; - apm_config.echo_canceller.enabled = true; + apm_config.echo_canceller.enabled = false; apm_config.echo_canceller.mobile_mode = false; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; @@ -250,8 +247,8 @@ void LLWebRTCImpl::init() processing_config.reverse_output_stream().set_num_channels(2); processing_config.reverse_output_stream().set_sample_rate_hz(48000); - apm->Initialize(processing_config); - apm->ApplyConfig(apm_config); + mAudioProcessingModule->Initialize(processing_config); + mAudioProcessingModule->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), @@ -262,7 +259,7 @@ void LLWebRTCImpl::init() nullptr /* video_encoder_factory */, nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, - apm); + mAudioProcessingModule); mWorkerThread->BlockingCall([this]() { mPeerDeviceModule->StartPlayout(); }); } @@ -318,6 +315,49 @@ void LLWebRTCImpl::setRecording(bool recording) }); } +void LLWebRTCImpl::setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config) +{ + webrtc::AudioProcessing::Config apm_config; + apm_config.echo_canceller.enabled = config.mEchoCancellation; + apm_config.echo_canceller.mobile_mode = false; + apm_config.gain_controller1.enabled = true; + apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; + apm_config.gain_controller2.enabled = true; + apm_config.high_pass_filter.enabled = true; + apm_config.transient_suppression.enabled = true; + apm_config.pipeline.multi_channel_render = true; + apm_config.pipeline.multi_channel_capture = true; + apm_config.pipeline.multi_channel_capture = true; + + switch (config.mNoiseSuppressionLevel) + { + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_NONE: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_LOW: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_MODERATE: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kHigh; + break; + case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_VERY_HIGH: + apm_config.noise_suppression.enabled = true; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh; + break; + default: + apm_config.noise_suppression.enabled = false; + apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow; + } + mAudioProcessingModule->ApplyConfig(apm_config); +} + void LLWebRTCImpl::refreshDevices() { mWorkerThread->PostTask([this]() { updateDevices(); }); @@ -616,32 +656,36 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } -// TODO: Add initialization structure through which -// stun and turn servers may be passed in from -// the sim or login. -bool LLWebRTCPeerConnectionImpl::initializeConnection() +bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options) { RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; mWebRTCImpl->PostSignalingTask( - [this]() + [this, options]() { + std::vector servers = options.mServers; + if(servers.empty()) + { + LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers; + ice_servers.mUrls.push_back("stun:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302"); + ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302"); + } + webrtc::PeerConnectionInterface::RTCConfiguration config; + for (auto server : servers) + { + webrtc::PeerConnectionInterface::IceServer ice_server; + ice_server.urls = server.mUrls; + ice_server.username = server.mUserName; + ice_server.password = server.mPassword; + config.servers.push_back(ice_server); + } + config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; - webrtc::PeerConnectionInterface::IceServer server; - server.uri = "stun:roxie-turn.staging.secondlife.io:3478"; - config.servers.push_back(server); - server.uri = "stun:stun.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun1.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun2.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun3.l.google.com:19302"; - config.servers.push_back(server); - server.uri = "stun:stun4.l.google.com:19302"; - config.servers.push_back(server); config.set_min_port(60000); config.set_max_port(60100); @@ -671,7 +715,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = true; // incompatible with opus stereo + audioOptions.echo_cancellation = false; // incompatible with opus stereo audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index be2e5cdf68..f3a33435c9 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -99,10 +99,32 @@ class LLWebRTCDevicesObserver // to enumerate, set, and get notifications of changes // for both capture (microphone) and render (speaker) // devices. + class LLWebRTCDeviceInterface { public: + struct AudioConfig { + bool mAGC { true }; + + bool mEchoCancellation { true }; + + // TODO: The various levels of noise suppression are configured + // on the APM which would require setting config on the APM. + // We should pipe the various values through + // later. + typedef enum { + NOISE_SUPPRESSION_LEVEL_NONE = 0, + NOISE_SUPPRESSION_LEVEL_LOW, + NOISE_SUPPRESSION_LEVEL_MODERATE, + NOISE_SUPPRESSION_LEVEL_HIGH, + NOISE_SUPPRESSION_LEVEL_VERY_HIGH + } ENoiseSuppressionLevel; + ENoiseSuppressionLevel mNoiseSuppressionLevel { NOISE_SUPPRESSION_LEVEL_VERY_HIGH }; + }; + + virtual void setAudioConfig(AudioConfig config) = 0; + // instructs webrtc to refresh the device list. virtual void refreshDevices() = 0; @@ -194,19 +216,35 @@ class LLWebRTCSignalingObserver virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; }; - // LLWebRTCPeerConnectionInterface representsd a connection to a peer, // in most cases a Secondlife WebRTC server. This interface // allows for management of this peer connection. class LLWebRTCPeerConnectionInterface { public: + + struct InitOptions + { + // equivalent of PeerConnectionInterface::IceServer + struct IceServers { + // Valid formats are described in RFC7064 and RFC7065. + // Urls should containe dns hostnames (not IP addresses) + // as the TLS certificate policy is 'secure.' + // and we do not currentply support TLS extensions. + std::vector mUrls; + std::string mUserName; + std::string mPassword; + }; + + std::vector mServers; + }; + + virtual bool initializeConnection(InitOptions options = InitOptions()) = 0; + virtual bool shutdownConnection() = 0; virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; - virtual bool initializeConnection() = 0; - virtual bool shutdownConnection() = 0; virtual void AnswerAvailable(const std::string &sdp) = 0; }; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 38810a29b5..32faf2516c 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -145,6 +145,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // LLWebRTCDeviceInterface // + void setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config = LLWebRTCDeviceInterface::AudioConfig()) override; + void refreshDevices() override; void setDevicesObserver(LLWebRTCDevicesObserver *observer) override; @@ -227,6 +229,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // The factory that allows creation of native webrtc PeerConnections. rtc::scoped_refptr mPeerConnectionFactory; + + rtc::scoped_refptr mAudioProcessingModule; // more native webrtc stuff std::unique_ptr mTaskQueueFactory; @@ -278,11 +282,11 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, // // LLWebRTCPeerConnection // + bool initializeConnection(InitOptions options = InitOptions()) override; + bool shutdownConnection() override; void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override; - bool initializeConnection() override; - bool shutdownConnection() override; void AnswerAvailable(const std::string &sdp) override; // diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 5f7d1d8a21..0de6db0d65 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -15108,6 +15108,39 @@ Value 44125 + VoiceEchoCancellation + + Comment + Voice Echo Cancellation + Persist + 1 + Type + Boolean + Value + 1 + + VoiceAutomaticGainControl + + Comment + Voice Automatic Gain Control + Persist + 1 + Type + Boolean + Value + 1 + + VoiceNoiseSuppressionLevel + + Comment + Voice Noise Suppression Level + Persist + 1 + Type + U32 + Value + 4 + WarningsAsChat Comment diff --git a/indra/newview/llviewercontrol.cpp b/indra/newview/llviewercontrol.cpp index 7738cb904e..44173a8043 100644 --- a/indra/newview/llviewercontrol.cpp +++ b/indra/newview/llviewercontrol.cpp @@ -803,6 +803,9 @@ void settings_setup_listeners() setting_setup_signal_listener(gSavedSettings, "PushToTalkButton", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "PushToTalkToggle", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceEarLocation", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceEchoCancellation", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceAutomaticGainControl", handleVoiceClientPrefsChanged); + setting_setup_signal_listener(gSavedSettings, "VoiceNoiseSuppressionLevel", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceInputAudioDevice", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "VoiceOutputAudioDevice", handleVoiceClientPrefsChanged); setting_setup_signal_listener(gSavedSettings, "AudioLevelMic", handleVoiceClientPrefsChanged); diff --git a/indra/newview/llvoicevivox.cpp b/indra/newview/llvoicevivox.cpp index 29aba8ecba..f5c9e3aa98 100644 --- a/indra/newview/llvoicevivox.cpp +++ b/indra/newview/llvoicevivox.cpp @@ -1154,7 +1154,9 @@ bool LLVivoxVoiceClient::provisionVoiceAccount() do { LLVoiceVivoxStats::getInstance()->provisionAttemptStart(); - result = httpAdapter->postAndSuspend(httpRequest, url, LLSD(), httpOpts); + LLSD body; + body["voice_server_type"] = "vivox"; + result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts); if (sShuttingDown) { diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index a8bc468f4c..bb5b386e7c 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -24,6 +24,7 @@ * $/LicenseInfo$ */ #include +#include #include "llvoicewebrtc.h" #include "llsdutil.h" @@ -279,7 +280,7 @@ void LLWebRTCVoiceClient::cleanUp() LL_DEBUGS("Voice") << "Exiting" << LL_ENDL; } -//--------------------------------------------------- +// -------------------------------------------------- const LLVoiceVersionInfo& LLWebRTCVoiceClient::getVersion() { @@ -299,6 +300,13 @@ void LLWebRTCVoiceClient::updateSettings() setRenderDevice(outputDevice); F32 mic_level = gSavedSettings.getF32("AudioLevelMic"); setMicGain(mic_level); + + llwebrtc::LLWebRTCDeviceInterface::AudioConfig config; + config.mEchoCancellation = gSavedSettings.getBOOL("VoiceEchoCancellation"); + config.mAGC = gSavedSettings.getBOOL("VoiceAutomaticGainControl"); + config.mNoiseSuppressionLevel = (llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel)gSavedSettings.getU32("VoiceNoiseSuppressionLevel"); + mWebRTCDeviceInterface->setAudioConfig(config); + } // Observers @@ -2480,6 +2488,28 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP); } +static llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions getConnectionOptions() +{ + llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions options; + llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions::IceServers servers; + + // TODO: Pull these from login + std::string grid = LLGridManager::getInstance()->getGridLoginID(); + std::transform(grid.begin(), grid.end(), grid.begin(), [](unsigned char c){ return std::tolower(c); }); + int num_servers = 2; + if (grid == "agni") + { + num_servers = 3; + } + for (int i=1; i <= num_servers; i++) + { + servers.mUrls.push_back(llformat("stun:stun%d.%s.secondlife.io:3478", i, grid.c_str())); + } + options.mServers.push_back(servers); + return options; +} + + // Primary state machine for negotiating a single voice connection to the // Secondlife WebRTC server. bool LLVoiceWebRTCConnection::connectionStateMachine() @@ -2498,10 +2528,11 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() mTrickling = false; mIceCompleted = false; setVoiceConnectionState(VOICE_STATE_WAIT_FOR_SESSION_START); + // tell the webrtc library that we want a connection. The library will // respond with an offer on a separate thread, which will cause // the session state to change. - if (!mWebRTCPeerConnectionInterface->initializeConnection()) + if (!mWebRTCPeerConnectionInterface->initializeConnection(getConnectionOptions())) { setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); } diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index 21fc79420b..96d39f3928 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -272,7 +272,7 @@ public: participantStatePtr_t findParticipantByID(const LLUUID& id); static ptr_t matchSessionByChannelID(const std::string& channel_id); - + void shutdownAllConnections(); void revive(); diff --git a/indra/newview/skins/default/xui/en/panel_preferences_sound.xml b/indra/newview/skins/default/xui/en/panel_preferences_sound.xml index ab2e9c72f3..3dba5d060b 100644 --- a/indra/newview/skins/default/xui/en/panel_preferences_sound.xml +++ b/indra/newview/skins/default/xui/en/panel_preferences_sound.xml @@ -340,7 +340,7 @@ follows="left|top" top_delta="-6" layout="topleft" - left_pad="20" + left_pad="10" width="360" height="40" name="media_ear_location"> @@ -422,7 +422,7 @@ control_name="VoiceEarLocation" follows="left|top" layout="topleft" - left_pad="20" + left_pad="10" top_delta="-6" width="360" height="40" @@ -454,18 +454,38 @@ name="enable_lip_sync" top_pad="10" width="237"/> + + + + Noise Suppression + + + + + + + + Date: Sat, 30 Mar 2024 22:03:30 -0700 Subject: [PATCH 11/21] Renegotiate on remote description error --- indra/llwebrtc/llwebrtc.cpp | 8 ++++++++ indra/newview/llvoicewebrtc.cpp | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a92b480e3a..bba1e99e2d 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -1073,6 +1073,10 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface * void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } } // @@ -1086,6 +1090,10 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError if (!error.ok()) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); + for (auto &observer : mSignalingObserverList) + { + observer->OnRenegotiationNeeded(); + } return; } mAnswerReceived = true; diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 72d343598b..c196e16684 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2733,8 +2733,8 @@ void LLVoiceWebRTCConnection::OnDataReceivedImpl(const std::string &data, bool b LLUUID agent_id(participant_id); if (agent_id.isNull()) { - LL_WARNS("Voice") << "Bad participant ID from data channel (" << participant_id << "):" << data << LL_ENDL; - continue; + // probably a test client. + continue; } LLWebRTCVoiceClient::participantStatePtr_t participant = From c6e673cda139f5faaa52ccd03a372e7ffa9f5716 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 30 Mar 2024 23:19:38 -0700 Subject: [PATCH 12/21] Fix windows crashes * sampling rate was set to 8khz for audio processing, which was causing a 'bands' mismatch with the echo cancler. * Some funnybusiness with lambdas and captures and such was causing a heap crash with respect to function parameters. --- indra/llwebrtc/llwebrtc.cpp | 32 ++++++++++++-------------------- indra/llwebrtc/llwebrtc.h | 3 ++- indra/llwebrtc/llwebrtc_impl.h | 2 +- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index a1e125a8f2..074b037529 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -239,16 +239,16 @@ void LLWebRTCImpl::init() webrtc::ProcessingConfig processing_config; processing_config.input_stream().set_num_channels(2); - processing_config.input_stream().set_sample_rate_hz(8000); + processing_config.input_stream().set_sample_rate_hz(48000); processing_config.output_stream().set_num_channels(2); - processing_config.output_stream().set_sample_rate_hz(8000); + processing_config.output_stream().set_sample_rate_hz(48000); processing_config.reverse_input_stream().set_num_channels(2); processing_config.reverse_input_stream().set_sample_rate_hz(48000); processing_config.reverse_output_stream().set_num_channels(2); processing_config.reverse_output_stream().set_sample_rate_hz(48000); - mAudioProcessingModule->Initialize(processing_config); mAudioProcessingModule->ApplyConfig(apm_config); + mAudioProcessingModule->Initialize(processing_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), @@ -659,35 +659,27 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve } } -bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options) + +bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnectionInterface::InitOptions& options) { RTC_DCHECK(!mPeerConnection); mAnswerReceived = false; mWebRTCImpl->PostSignalingTask( - [this, options]() + [this,options]() { - std::vector servers = options.mServers; - if(servers.empty()) - { - LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers; - ice_servers.mUrls.push_back("stun:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302"); - ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302"); - } - webrtc::PeerConnectionInterface::RTCConfiguration config; - for (auto server : servers) + for (auto server : options.mServers) { webrtc::PeerConnectionInterface::IceServer ice_server; - ice_server.urls = server.mUrls; + for (auto url : server.mUrls) + { + ice_server.urls.push_back(url); + } ice_server.username = server.mUserName; ice_server.password = server.mPassword; config.servers.push_back(ice_server); } - config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; config.set_min_port(60000); @@ -718,7 +710,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInte cricket::AudioOptions audioOptions; audioOptions.auto_gain_control = true; - audioOptions.echo_cancellation = false; // incompatible with opus stereo + audioOptions.echo_cancellation = true; audioOptions.noise_suppression = true; mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream"); diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index f3a33435c9..8830799cde 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -227,6 +227,7 @@ class LLWebRTCPeerConnectionInterface { // equivalent of PeerConnectionInterface::IceServer struct IceServers { + // Valid formats are described in RFC7064 and RFC7065. // Urls should containe dns hostnames (not IP addresses) // as the TLS certificate policy is 'secure.' @@ -239,7 +240,7 @@ class LLWebRTCPeerConnectionInterface std::vector mServers; }; - virtual bool initializeConnection(InitOptions options = InitOptions()) = 0; + virtual bool initializeConnection(const InitOptions& options) = 0; virtual bool shutdownConnection() = 0; virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0; diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 32faf2516c..328e962c50 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -282,7 +282,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface, // // LLWebRTCPeerConnection // - bool initializeConnection(InitOptions options = InitOptions()) override; + bool initializeConnection(const InitOptions& options) override; bool shutdownConnection() override; void setSignalingObserver(LLWebRTCSignalingObserver *observer) override; From c826aea079c59950a4064a94825534884fed8bf8 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 1 Apr 2024 21:39:17 -0700 Subject: [PATCH 13/21] Fix "default" audio device handling. Windows and Mac/Linux behave slightly differently with respect to Default devices, in that mac/linux (I think) simply assumes the device at index 0 is the default one, and windows has a separate API for enabling the default device. --- indra/llwebrtc/llwebrtc.cpp | 228 +++++++++++++++++++------------- indra/llwebrtc/llwebrtc_impl.h | 6 +- indra/newview/llvoicewebrtc.cpp | 66 +++++---- 3 files changed, 172 insertions(+), 128 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 074b037529..75b2332bed 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -40,6 +40,11 @@ namespace llwebrtc { +static int16_t PLAYOUT_DEVICE_DEFAULT = -1; +static int16_t PLAYOUT_DEVICE_BAD = -2; +static int16_t RECORD_DEVICE_DEFAULT = -1; +static int16_t RECORD_DEVICE_BAD = -2; + LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEnergy(0.0) {} float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; } @@ -189,10 +194,16 @@ void LLWebRTCImpl::init() std::unique_ptr(mTuningAudioDeviceObserver)); mTuningDeviceModule->Init(); - mTuningDeviceModule->SetStereoRecording(true); - mTuningDeviceModule->SetStereoPlayout(true); + mTuningDeviceModule->SetPlayoutDevice(mPlayoutDevice); + mTuningDeviceModule->SetRecordingDevice(mRecordingDevice); mTuningDeviceModule->EnableBuiltInAEC(false); mTuningDeviceModule->SetAudioDeviceSink(this); + mTuningDeviceModule->InitMicrophone(); + mTuningDeviceModule->InitSpeaker(); + mTuningDeviceModule->InitRecording(); + mTuningDeviceModule->InitPlayout(); + mTuningDeviceModule->SetStereoRecording(true); + mTuningDeviceModule->SetStereoPlayout(true); updateDevices(); }); @@ -207,13 +218,13 @@ void LLWebRTCImpl::init() mPeerDeviceModule->Init(); mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); - mPeerDeviceModule->SetStereoRecording(true); - mPeerDeviceModule->SetStereoPlayout(true); mPeerDeviceModule->EnableBuiltInAEC(false); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->SetStereoRecording(true); + mPeerDeviceModule->SetStereoPlayout(true); }); // The custom processor allows us to retrieve audio data (and levels) @@ -375,145 +386,169 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) } } -static int16_t ll_get_device_module_capture_device(rtc::scoped_refptr device_module, const std::string &id) -{ - int16_t recordingDevice = 0; - int16_t captureDeviceCount = device_module->RecordingDevices(); - for (int16_t i = 0; i < captureDeviceCount; i++) - { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - device_module->RecordingDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default - { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - recordingDevice = i; - break; - } - } - return recordingDevice; -} - void ll_set_device_module_capture_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopRecording(); - device_module->SetRecordingDevice(device); +#if LL_WINDOWS + if (device < 0) + { + device_module->SetRecordingDevice(webrtc::AudioDeviceModule::kDefaultDevice); + } + else + { + device_module->SetRecordingDevice(device); + } +#else + // passed in default is -1, but the device list + // has it at 0 + device_module->SetPlayoutDevice(device + 1); +#endif device_module->InitMicrophone(); - device_module->SetStereoRecording(false); device_module->InitRecording(); + device_module->SetStereoRecording(false); device_module->StartRecording(); } void LLWebRTCImpl::setCaptureDevice(const std::string &id) { - - mWorkerThread->PostTask( - [this, id]() - { - int16_t recordingDevice = ll_get_device_module_capture_device(mTuningDeviceModule, id); - if (recordingDevice != mRecordingDevice) - { - mRecordingDevice = recordingDevice; - if (mTuningMode) - { - ll_set_device_module_capture_device(mTuningDeviceModule, recordingDevice); - } - else - { - ll_set_device_module_capture_device(mPeerDeviceModule, recordingDevice); - } - } - }); -} - -static int16_t ll_get_device_module_render_device( - rtc::scoped_refptr device_module, - const std::string &id) -{ - int16_t playoutDevice = 0; - int16_t playoutDeviceCount = device_module->PlayoutDevices(); - for (int16_t i = 0; i < playoutDeviceCount; i++) + int16_t recordingDevice = RECORD_DEVICE_DEFAULT; + if (id != "Default") { - char name[webrtc::kAdmMaxDeviceNameSize]; - char guid[webrtc::kAdmMaxGuidSize]; - device_module->PlayoutDeviceName(i, name, guid); - if (id == guid || id == "Default") // first one in list is default + for (int16_t i = 0; i < mRecordingDeviceList.size(); i++) { - RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i; - playoutDevice = i; - break; + if (mRecordingDeviceList[i].mID == id) + { + recordingDevice = i; + break; + } } } - return playoutDevice; + if (recordingDevice == mRecordingDevice) + { + return; + } + mRecordingDevice = recordingDevice; + if (mTuningMode) + { + mWorkerThread->PostTask([this, recordingDevice]() { ll_set_device_module_capture_device(mTuningDeviceModule, recordingDevice); }); + } + else + { + mWorkerThread->PostTask([this, recordingDevice]() { ll_set_device_module_capture_device(mPeerDeviceModule, recordingDevice); }); + } } void ll_set_device_module_render_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopPlayout(); - device_module->SetPlayoutDevice(device); +#if LL_WINDOWS + if (device < 0) + { + device_module->SetPlayoutDevice(webrtc::AudioDeviceModule::kDefaultDevice); + } + else + { + device_module->SetPlayoutDevice(device); + } +#else + device_module->SetPlayoutDevice(device + 1); +#endif device_module->InitSpeaker(); - device_module->SetStereoPlayout(false); device_module->InitPlayout(); - device_module->StartPlayout(); + device_module->SetStereoPlayout(true); } void LLWebRTCImpl::setRenderDevice(const std::string &id) { - mWorkerThread->PostTask( - [this, id]() - { - int16_t playoutDevice = ll_get_device_module_render_device(mTuningDeviceModule, id); - if (playoutDevice != mPlayoutDevice) + int16_t playoutDevice = PLAYOUT_DEVICE_DEFAULT; + if (id != "Default") + { + for (int16_t i = 0; i < mPlayoutDeviceList.size(); i++) + { + if (mPlayoutDeviceList[i].mID == id) { - mPlayoutDevice = playoutDevice; - if (mTuningMode) - { - ll_set_device_module_render_device(mTuningDeviceModule, playoutDevice); - } - else - { - ll_set_device_module_render_device(mPeerDeviceModule, playoutDevice); - } + playoutDevice = i; + break; } - }); + } + } + if (playoutDevice == mPlayoutDevice) + { + return; + } + mPlayoutDevice = playoutDevice; + + if (mTuningMode) + { + mWorkerThread->PostTask( + [this, playoutDevice]() + { + ll_set_device_module_render_device(mTuningDeviceModule, playoutDevice); + }); + } + else + { + mWorkerThread->PostTask( + [this, playoutDevice]() + { + ll_set_device_module_render_device(mPeerDeviceModule, playoutDevice); + mPeerDeviceModule->StartPlayout(); + }); + } } // updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - int16_t currentRenderDeviceIndex = mTuningDeviceModule->GetPlayoutDevice(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); - LLWebRTCVoiceDeviceList renderDeviceList; - for (int16_t index = 0; index < renderDeviceCount; index++) + mPlayoutDeviceList.clear(); +#if LL_WINDOWS + int16_t index = 0; +#else + // index zero is always "Default" for darwin/linux, + // which is a special case, so skip it. + int16_t index = 1; +#endif + for (; index < renderDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - renderDeviceList.emplace_back(name, guid); + mPlayoutDeviceList.emplace_back(name, guid); } - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - int16_t currentCaptureDeviceIndex = mTuningDeviceModule->GetRecordingDevice(); + int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); - LLWebRTCVoiceDeviceList captureDeviceList; - for (int16_t index = 0; index < captureDeviceCount; index++) + mRecordingDeviceList.clear(); +#if LL_WINDOWS + index = 0; +#else + // index zero is always "Default" for darwin/linux, + // which is a special case, so skip it. + index = 1; +#endif + for (; index < captureDeviceCount; index++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); - captureDeviceList.emplace_back(name, guid); + mRecordingDeviceList.emplace_back(name, guid); } + for (auto &observer : mVoiceDevicesObserverList) { - observer->OnDevicesChanged(renderDeviceList, - captureDeviceList); + observer->OnDevicesChanged(mPlayoutDeviceList, mRecordingDeviceList); } } void LLWebRTCImpl::OnDevicesUpdated() { + // reset these to a bad value so an update is forced + mRecordingDevice = RECORD_DEVICE_BAD; + mPlayoutDevice = PLAYOUT_DEVICE_BAD; + updateDevices(); } @@ -529,17 +564,24 @@ void LLWebRTCImpl::setTuningMode(bool enable) mPeerDeviceModule->StopPlayout(); ll_set_device_module_render_device(mTuningDeviceModule, mPlayoutDevice); ll_set_device_module_capture_device(mTuningDeviceModule, mRecordingDevice); + mTuningDeviceModule->InitPlayout(); + mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); - mTuningDeviceModule->StartPlayout(); + // TODO: Starting Playout on the TDM appears to create an audio artifact (click) + // in this case, so disabling it for now. We may have to do something different + // if we enable 'echo playback' via the TDM when tuning. + //mTuningDeviceModule->StartPlayout(); } else { mTuningDeviceModule->StopRecording(); - mTuningDeviceModule->StopPlayout(); + //mTuningDeviceModule->StopPlayout(); ll_set_device_module_render_device(mPeerDeviceModule, mPlayoutDevice); ll_set_device_module_capture_device(mPeerDeviceModule, mRecordingDevice); - mPeerDeviceModule->StartRecording(); + mPeerDeviceModule->InitPlayout(); + mPeerDeviceModule->InitRecording(); mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); } } ); diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 328e962c50..78ae6b4444 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -244,8 +244,12 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS // accessors in native webrtc for devices aren't apparently implemented yet. bool mTuningMode; - int32_t mPlayoutDevice; int32_t mRecordingDevice; + LLWebRTCVoiceDeviceList mRecordingDeviceList; + + int32_t mPlayoutDevice; + LLWebRTCVoiceDeviceList mPlayoutDeviceList; + bool mMute; LLAudioDeviceObserver * mTuningAudioDeviceObserver; diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index f3d3460022..3cc4aa113c 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -294,19 +294,31 @@ void LLWebRTCVoiceClient::updateSettings() LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE setVoiceEnabled(LLVoiceClient::getInstance()->voiceEnabled()); - setEarLocation(gSavedSettings.getS32("VoiceEarLocation")); + static LLCachedControl sVoiceEarLocation(gSavedSettings, "VoiceEarLocation"); + setEarLocation(sVoiceEarLocation); + + static LLCachedControl sInputDevice(gSavedSettings, "VoiceInputAudioDevice"); + setCaptureDevice(sInputDevice); + + static LLCachedControl sOutputDevice(gSavedSettings, "VoiceOutputAudioDevice"); + setRenderDevice(sOutputDevice); + + static LLCachedControl sMicLevel(gSavedSettings, "AudioLevelMic"); + setMicGain(sMicLevel); - std::string inputDevice = gSavedSettings.getString("VoiceInputAudioDevice"); - setCaptureDevice(inputDevice); - std::string outputDevice = gSavedSettings.getString("VoiceOutputAudioDevice"); - setRenderDevice(outputDevice); - F32 mic_level = gSavedSettings.getF32("AudioLevelMic"); - setMicGain(mic_level); - llwebrtc::LLWebRTCDeviceInterface::AudioConfig config; - config.mEchoCancellation = gSavedSettings.getBOOL("VoiceEchoCancellation"); - config.mAGC = gSavedSettings.getBOOL("VoiceAutomaticGainControl"); - config.mNoiseSuppressionLevel = (llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel)gSavedSettings.getU32("VoiceNoiseSuppressionLevel"); + + static LLCachedControl sEchoCancellation(gSavedSettings, "VoiceEchoCancellation", true); + config.mEchoCancellation = sEchoCancellation; + + static LLCachedControl sAGC(gSavedSettings, "VoiceAutomaticGainControl", true); + config.mAGC = sAGC; + + static LLCachedControl sNoiseSuppressionLevel(gSavedSettings, + "VoiceNoiseSuppressionLevel", + llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel::NOISE_SUPPRESSION_LEVEL_VERY_HIGH); + config.mNoiseSuppressionLevel = (llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel) (U32)sNoiseSuppressionLevel; + mWebRTCDeviceInterface->setAudioConfig(config); } @@ -607,6 +619,7 @@ void LLWebRTCVoiceClient::setDevicesListUpdated(bool state) void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceList& render_devices, const llwebrtc::LLWebRTCVoiceDeviceList& capture_devices) { + LL::WorkQueue::postMaybe(mMainQueue, [=] { @@ -618,43 +631,25 @@ void LLWebRTCVoiceClient::OnDevicesChangedImpl(const llwebrtc::LLWebRTCVoiceDevi const llwebrtc::LLWebRTCVoiceDeviceList &capture_devices) { LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE - std::string inputDevice = gSavedSettings.getString("VoiceInputAudioDevice"); std::string outputDevice = gSavedSettings.getString("VoiceOutputAudioDevice"); LL_DEBUGS("Voice") << "Setting devices to-input: '" << inputDevice << "' output: '" << outputDevice << "'" << LL_ENDL; clearRenderDevices(); - bool renderDeviceSet = false; for (auto &device : render_devices) { addRenderDevice(LLVoiceDevice(device.mDisplayName, device.mID)); - LL_DEBUGS("Voice") << "Checking render device" << "'" << device.mID << "'" << LL_ENDL; - if (outputDevice == device.mID) - { - renderDeviceSet = true; - } - } - if (!renderDeviceSet) - { - setRenderDevice("Default"); } + setRenderDevice(outputDevice); clearCaptureDevices(); - bool captureDeviceSet = false; for (auto &device : capture_devices) { LL_DEBUGS("Voice") << "Checking capture device:'" << device.mID << "'" << LL_ENDL; addCaptureDevice(LLVoiceDevice(device.mDisplayName, device.mID)); - if (inputDevice == device.mID) - { - captureDeviceSet = true; - } - } - if (!captureDeviceSet) - { - setCaptureDevice("Default"); } + setCaptureDevice(inputDevice); setDevicesListUpdated(true); } @@ -1559,10 +1554,13 @@ F32 LLWebRTCVoiceClient::getUserVolume(const LLUUID& id) // Minimum volume will be returned for users with voice disabled F32 result = LLVoiceClient::VOLUME_MIN; - participantStatePtr_t participant(mSession->findParticipantByID(id)); - if(participant) + if (mSession) { - result = participant->mVolume; + participantStatePtr_t participant(mSession->findParticipantByID(id)); + if (participant) + { + result = participant->mVolume; + } } return result; From fdf0fbce5b4c8f15493ff064ef4d092f52c6ef73 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 1 Apr 2024 22:39:59 -0700 Subject: [PATCH 14/21] oopse --- indra/llwebrtc/llwebrtc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 75b2332bed..7570bb4434 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -401,7 +401,7 @@ void ll_set_device_module_capture_device(rtc::scoped_refptrSetPlayoutDevice(device + 1); + device_module->SetRecordingDevice(device + 1); #endif device_module->InitMicrophone(); device_module->InitRecording(); From 3ff1f0f9518c7b2faf1f9ea5e3f2eb42539ba2e5 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 4 Apr 2024 11:32:51 -0700 Subject: [PATCH 15/21] An explicit "decline" message for P2P When declining a P2P voice call for webrtc, instead of relying on vivox to stop "ringing," we need to send an explicit decline message from the peer through the server infrastructure. --- indra/newview/llimview.cpp | 41 ++++++++++++++++++++++++++++++--- indra/newview/llvoicevivox.cpp | 2 +- indra/newview/llvoicewebrtc.cpp | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp index 215cc4103b..2f2a6041ad 100644 --- a/indra/newview/llimview.cpp +++ b/indra/newview/llimview.cpp @@ -103,6 +103,7 @@ enum EMultiAgentChatSessionType void startConferenceCoro(std::string url, LLUUID tempSessionId, LLUUID creatorId, LLUUID otherParticipantId, LLSD agents); void startP2PCoro(std::string url, LLUUID tempSessionId, LLUUID creatorId, LLUUID otherParticipantId); +void declineP2PCoro(std::string url, LLUUID sessionID); void chatterBoxInvitationCoro(std::string url, LLUUID sessionId, LLIMMgr::EInvitationType invitationType); void chatterBoxHistoryCoro(std::string url, LLUUID sessionId, std::string from, std::string message, U32 timestamp); @@ -488,6 +489,27 @@ void startP2PCoro(std::string url, LLUUID sessionID, LLUUID creatorId, LLUUID ot } } +void declineP2PCoro(std::string url, LLUUID sessionID) +{ + LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("ConferenceChatStart", httpPolicy)); + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + + LLSD postData; + postData["method"] = "decline p2p"; + postData["session-id"] = sessionID; + + LLSD result = httpAdapter->postAndSuspend(httpRequest, url, postData); + + LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; + LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); + + if (!status) + { + LL_WARNS("LLIMModel") << "Failed to decline p2p session:" << postData << "->" << result << LL_ENDL; + } +} + void chatterBoxInvitationCoro(std::string url, LLUUID sessionId, LLIMMgr::EInvitationType invitationType) { LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); @@ -3005,10 +3027,23 @@ void LLIncomingCallDialog::processCallResponse(S32 response, const LLSD &payload { if (type == IM_SESSION_P2P_INVITE) { - LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(payload["voice_session_info"]); - if (call) + // create a normal IM session + LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(payload["voice_channel_info"]); + if (call) + { + call->declineInvite(); + } + else { - call->declineInvite(); + // webrtc-style decline. + LLViewerRegion *region = gAgent.getRegion(); + if (region) + { + std::string url = region->getCapability("ChatSessionRequest"); + LLCoros::instance().launch("declineP2P", + boost::bind(&declineP2PCoro, url, session_id)); + } + } } else diff --git a/indra/newview/llvoicevivox.cpp b/indra/newview/llvoicevivox.cpp index f5c9e3aa98..be9e055203 100644 --- a/indra/newview/llvoicevivox.cpp +++ b/indra/newview/llvoicevivox.cpp @@ -5089,7 +5089,7 @@ bool LLVivoxVoiceClient::isCurrentChannel(const LLSD &channelInfo) } if (mAudioSession) { - if (!channelInfo["sessionHandle"].asString().empty()) + if (!channelInfo["session_handle"].asString().empty()) { return mAudioSession->mHandle == channelInfo["session_handle"].asString(); } diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 3cc4aa113c..dbfc28c44f 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -1360,7 +1360,7 @@ bool LLWebRTCVoiceClient::isCurrentChannel(const LLSD &channelInfo) if (mSession) { - if (!channelInfo["sessionHandle"].asString().empty()) + if (!channelInfo["session_handle"].asString().empty()) { return mSession->mHandle == channelInfo["session_handle"].asString(); } From b6f4cec7d5f771df3ed5e8f0b241b2212a6e9ffa Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Thu, 4 Apr 2024 16:13:33 -0700 Subject: [PATCH 16/21] checkpoint p2p im decline/accept work --- indra/newview/llimview.cpp | 85 ++++++++++++++++++---------------- indra/newview/llvoicechannel.h | 2 + 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp index 2f2a6041ad..ff90d75ad2 100644 --- a/indra/newview/llimview.cpp +++ b/indra/newview/llimview.cpp @@ -102,8 +102,7 @@ enum EMultiAgentChatSessionType void startConferenceCoro(std::string url, LLUUID tempSessionId, LLUUID creatorId, LLUUID otherParticipantId, LLSD agents); -void startP2PCoro(std::string url, LLUUID tempSessionId, LLUUID creatorId, LLUUID otherParticipantId); -void declineP2PCoro(std::string url, LLUUID sessionID); +void startP2PVoiceCoro(std::string url, LLUUID tempSessionId, LLUUID creatorId, LLUUID otherParticipantId); void chatterBoxInvitationCoro(std::string url, LLUUID sessionId, LLIMMgr::EInvitationType invitationType); void chatterBoxHistoryCoro(std::string url, LLUUID sessionId, std::string from, std::string message, U32 timestamp); @@ -457,14 +456,14 @@ void startConferenceCoro(std::string url, } } -void startP2PCoro(std::string url, LLUUID sessionID, LLUUID creatorId, LLUUID otherParticipantId) +void startP2PVoiceCoro(std::string url, LLUUID sessionID, LLUUID creatorId, LLUUID otherParticipantId) { LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); - LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("ConferenceChatStart", httpPolicy)); + LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("StartP2PVoiceCoro", httpPolicy)); LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); LLSD postData; - postData["method"] = "start p2p"; + postData["method"] = "start p2p voice"; postData["session-id"] = sessionID; postData["params"] = otherParticipantId; LLSD altParams; @@ -489,27 +488,6 @@ void startP2PCoro(std::string url, LLUUID sessionID, LLUUID creatorId, LLUUID ot } } -void declineP2PCoro(std::string url, LLUUID sessionID) -{ - LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); - LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("ConferenceChatStart", httpPolicy)); - LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); - - LLSD postData; - postData["method"] = "decline p2p"; - postData["session-id"] = sessionID; - - LLSD result = httpAdapter->postAndSuspend(httpRequest, url, postData); - - LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS]; - LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults); - - if (!status) - { - LL_WARNS("LLIMModel") << "Failed to decline p2p session:" << postData << "->" << result << LL_ENDL; - } -} - void chatterBoxInvitationCoro(std::string url, LLUUID sessionId, LLIMMgr::EInvitationType invitationType) { LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); @@ -2147,7 +2125,7 @@ bool LLIMModel::sendStartSession( if (region) { std::string url = region->getCapability("ChatSessionRequest"); - LLCoros::instance().launch("startP2P", boost::bind(&startP2PCoro, url, temp_session_id, gAgent.getID(), other_participant_id)); + LLCoros::instance().launch("startP2PVoiceCoro", boost::bind(&startP2PVoiceCoro, url, temp_session_id, gAgent.getID(), other_participant_id)); } return true; } @@ -3027,7 +3005,8 @@ void LLIncomingCallDialog::processCallResponse(S32 response, const LLSD &payload { if (type == IM_SESSION_P2P_INVITE) { - // create a normal IM session + // decline p2p voice, either via the vivox-style call mechanism + // or via the webrtc-style "decline p2p" mechanism. LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(payload["voice_channel_info"]); if (call) { @@ -3040,24 +3019,28 @@ void LLIncomingCallDialog::processCallResponse(S32 response, const LLSD &payload if (region) { std::string url = region->getCapability("ChatSessionRequest"); - LLCoros::instance().launch("declineP2P", - boost::bind(&declineP2PCoro, url, session_id)); - } + LLSD data; + data["method"] = "decline p2p voice"; + data["session-id"] = session_id; + + LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "P2P declined", "P2P decline failed."); + } } } else { - std::string url = gAgent.getRegion()->getCapability( - "ChatSessionRequest"); + LLViewerRegion *region = gAgent.getRegion(); + if (region) + { + std::string url = region->getCapability("ChatSessionRequest"); - LLSD data; - data["method"] = "decline invitation"; - data["session-id"] = session_id; + LLSD data; + data["method"] = "decline invitation"; + data["session-id"] = session_id; - LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, - "Invitation declined", - "Invitation decline failed."); + LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "Invitation declined", "Invitation decline failed."); + } } } @@ -3607,6 +3590,30 @@ void LLIMMgr::clearPendingInvitation(const LLUUID& session_id) void LLIMMgr::processAgentListUpdates(const LLUUID& session_id, const LLSD& body) { + if (body.isMap() && body.has("agent_updates") && body["agent_updates"].isMap()) + { + LLSD::map_const_iterator update_it; + for (update_it = body["agent_updates"].beginMap(); update_it != body["agent_updates"].endMap(); ++update_it) + { + LLUUID agent_id(update_it->first); + LLSD agent_data = update_it->second; + + if (agent_data.isMap()) + { + // Is one of the participants leaving a P2P Chat? + if (agent_data.has("transition") && agent_data["transition"].asString() == "LEAVE") + { + LLVoiceChannelGroup *channelp = dynamic_cast < LLVoiceChannelGroup*>(LLVoiceChannel::getChannelByID(session_id)); + if (channelp && channelp->isP2P()) + { + endCall(session_id); + break; + } + } + } + } + } + LLFloaterIMSession* im_floater = LLFloaterIMSession::findInstance(session_id); if ( im_floater ) { diff --git a/indra/newview/llvoicechannel.h b/indra/newview/llvoicechannel.h index bc22bf0df6..adc387e22d 100644 --- a/indra/newview/llvoicechannel.h +++ b/indra/newview/llvoicechannel.h @@ -147,6 +147,8 @@ public: void setChannelInfo(const LLSD &channelInfo) override; void requestChannelInfo() override; + bool isP2P() { return mIsP2P; } + protected: void setState(EState state) override; From 4ecf050439da2d16028ce3018a66424b0434005c Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sat, 6 Apr 2024 21:58:17 -0700 Subject: [PATCH 17/21] Tell the user the peer has declined and shut down voice channel When the server sends up a notification that a peer is not doing voice in adhoc-style p2p chat, shut down the voice call and notify the user the peer has declined. --- indra/newview/llimview.cpp | 10 +++++++--- indra/newview/llvoicewebrtc.cpp | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp index ff90d75ad2..f710fea3e0 100644 --- a/indra/newview/llimview.cpp +++ b/indra/newview/llimview.cpp @@ -3595,17 +3595,21 @@ void LLIMMgr::processAgentListUpdates(const LLUUID& session_id, const LLSD& body LLSD::map_const_iterator update_it; for (update_it = body["agent_updates"].beginMap(); update_it != body["agent_updates"].endMap(); ++update_it) { - LLUUID agent_id(update_it->first); LLSD agent_data = update_it->second; - if (agent_data.isMap()) + if (agent_data.isMap() && agent_data.has("info") && agent_data["info"].isMap()) { // Is one of the participants leaving a P2P Chat? - if (agent_data.has("transition") && agent_data["transition"].asString() == "LEAVE") + if (agent_data["info"].has("can_voice_chat") && !agent_data["info"]["can_voice_chat"].asBoolean()) { LLVoiceChannelGroup *channelp = dynamic_cast < LLVoiceChannelGroup*>(LLVoiceChannel::getChannelByID(session_id)); if (channelp && channelp->isP2P()) { + // it's an adhoc-style P2P channel, and voice is disabled / declined. notify the user + // and shut down the voice channel. + LLSD notifyArgs = LLSD::emptyMap(); + notifyArgs["VOICE_CHANNEL_NAME"] = channelp->getSessionName(); + LLNotificationsUtil::add("P2PCallDeclined", notifyArgs); endCall(session_id); break; } diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index dbfc28c44f..cb01a079cc 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2368,12 +2368,14 @@ void LLVoiceWebRTCConnection::breakVoiceConnectionCoro() if (!regionp || !regionp->capabilitiesReceived()) { LL_DEBUGS("Voice") << "no capabilities for voice provisioning; waiting " << LL_ENDL; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); return; } std::string url = regionp->getCapability("ProvisionVoiceAccountRequest"); if (url.empty()) { + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); return; } From 34ed990fa363f3df0a9817d36ef0becdcf641e80 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 7 Apr 2024 14:13:00 -0700 Subject: [PATCH 18/21] Show 'decline' when peer declines p2p voice The simulator will send a chatterbox notification that voice is no longer in use for a given channel, and the viewer should take that as a case where the peer does not want voice, hence it's a decline. --- indra/llwebrtc/llwebrtc.cpp | 30 ++++++++++++++++++++++++------ indra/llwebrtc/llwebrtc_impl.h | 1 + 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 7570bb4434..57130bb734 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -26,7 +26,6 @@ #include "llwebrtc_impl.h" #include -#include #include #include "api/audio_codecs/audio_decoder_factory.h" @@ -389,7 +388,7 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer) void ll_set_device_module_capture_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopRecording(); -#if LL_WINDOWS +#if WEBRTC_WIN if (device < 0) { device_module->SetRecordingDevice(webrtc::AudioDeviceModule::kDefaultDevice); @@ -442,7 +441,7 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) void ll_set_device_module_render_device(rtc::scoped_refptr device_module, int16_t device) { device_module->StopPlayout(); -#if LL_WINDOWS +#if WEBRTC_WIN if (device < 0) { device_module->SetPlayoutDevice(webrtc::AudioDeviceModule::kDefaultDevice); @@ -501,10 +500,10 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id) // updateDevices needs to happen on the worker thread. void LLWebRTCImpl::updateDevices() { - int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); + int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices(); mPlayoutDeviceList.clear(); -#if LL_WINDOWS +#if WEBRTC_WIN int16_t index = 0; #else // index zero is always "Default" for darwin/linux, @@ -516,13 +515,23 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); + +#if WEBRTC_LINUX + // Linux audio implementation (pulse and alsa) + // return empty strings for the guid, so + // use the name for the guid + if (!strcmp(guid, "")) + { + strcpy(guid, name); + } +#endif // WEBRTC_LINUX mPlayoutDeviceList.emplace_back(name, guid); } int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); mRecordingDeviceList.clear(); -#if LL_WINDOWS +#if WEBRTC_WIN index = 0; #else // index zero is always "Default" for darwin/linux, @@ -534,6 +543,15 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); +#if WEBRTC_LINUX + // Linux audio implementation (pulse and alsa) + // return empty strings for the guid, so + // use the name for the guid + if (!strcmp(guid, "")) + { + strcpy(guid, name); + } +#endif // WEBRTC_LINUX mRecordingDeviceList.emplace_back(name, guid); } diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h index 78ae6b4444..e1031099c7 100644 --- a/indra/llwebrtc/llwebrtc_impl.h +++ b/indra/llwebrtc/llwebrtc_impl.h @@ -35,6 +35,7 @@ #define WEBRTC_POSIX 1 #elif __linux__ #define WEBRTC_LINUX 1 +#define WEBRTC_POSIX 1 #endif #include "llwebrtc.h" From 4be63d8c1f56fb6d00f13b9a2d84f2d58e170344 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 7 Apr 2024 14:16:07 -0700 Subject: [PATCH 19/21] missed a file --- indra/newview/llimview.cpp | 58 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp index f710fea3e0..f01fde08d9 100644 --- a/indra/newview/llimview.cpp +++ b/indra/newview/llimview.cpp @@ -2727,7 +2727,7 @@ LLIncomingCallDialog::LLIncomingCallDialog(const LLSD& payload) : void LLIncomingCallDialog::onLifetimeExpired() { - LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(mPayload["voice_session_info"]); + LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(mPayload["voice_channel_info"]); if (call) { // restart notification's timer if call is still valid @@ -3590,33 +3590,39 @@ void LLIMMgr::clearPendingInvitation(const LLUUID& session_id) void LLIMMgr::processAgentListUpdates(const LLUUID& session_id, const LLSD& body) { - if (body.isMap() && body.has("agent_updates") && body["agent_updates"].isMap()) - { - LLSD::map_const_iterator update_it; - for (update_it = body["agent_updates"].beginMap(); update_it != body["agent_updates"].endMap(); ++update_it) - { - LLSD agent_data = update_it->second; + if (body.isMap() && body.has("agent_updates") && body["agent_updates"].isMap()) + { + LLSD::map_const_iterator update_it; + for (update_it = body["agent_updates"].beginMap(); update_it != body["agent_updates"].endMap(); ++update_it) + { + LLUUID agent_id = LLUUID(update_it->first); + LLSD agent_data = update_it->second; + if (agent_data.has("transition") && agent_data["transition"].asString() == "LEAVE") + { + // ignore actual leaves as those will be handled separately. + continue; + } - if (agent_data.isMap() && agent_data.has("info") && agent_data["info"].isMap()) - { - // Is one of the participants leaving a P2P Chat? - if (agent_data["info"].has("can_voice_chat") && !agent_data["info"]["can_voice_chat"].asBoolean()) - { - LLVoiceChannelGroup *channelp = dynamic_cast < LLVoiceChannelGroup*>(LLVoiceChannel::getChannelByID(session_id)); - if (channelp && channelp->isP2P()) - { - // it's an adhoc-style P2P channel, and voice is disabled / declined. notify the user + if (agent_id != gAgentID && agent_data.isMap() && agent_data.has("info") && agent_data["info"].isMap()) + { + // Is one of the participants leaving a P2P Chat? + if (agent_data["info"].has("can_voice_chat") && !agent_data["info"]["can_voice_chat"].asBoolean()) + { + LLVoiceChannelGroup *channelp = dynamic_cast < LLVoiceChannelGroup*>(LLVoiceChannel::getChannelByID(session_id)); + if (channelp && channelp->isP2P()) + { + // it's an adhoc-style P2P channel, and the peer has declined voice. notify the user // and shut down the voice channel. - LLSD notifyArgs = LLSD::emptyMap(); - notifyArgs["VOICE_CHANNEL_NAME"] = channelp->getSessionName(); - LLNotificationsUtil::add("P2PCallDeclined", notifyArgs); - endCall(session_id); - break; - } - } - } - } - } + LLSD notifyArgs = LLSD::emptyMap(); + notifyArgs["VOICE_CHANNEL_NAME"] = channelp->getSessionName(); + LLNotificationsUtil::add("P2PCallDeclined", notifyArgs); + endCall(session_id); + break; + } + } + } + } + } LLFloaterIMSession* im_floater = LLFloaterIMSession::findInstance(session_id); if ( im_floater ) From 648741470f67120d99aec3a4aeceeaf4395cca09 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Sun, 7 Apr 2024 19:23:00 -0700 Subject: [PATCH 20/21] CR suggestions --- indra/llwebrtc/llwebrtc.cpp | 19 --------------- indra/llwebrtc/llwebrtc.h | 7 +++++- indra/newview/llimview.cpp | 48 ++++++++++++++++++------------------- 3 files changed, 30 insertions(+), 44 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 57130bb734..eb9bb65e67 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -515,16 +515,6 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->PlayoutDeviceName(index, name, guid); - -#if WEBRTC_LINUX - // Linux audio implementation (pulse and alsa) - // return empty strings for the guid, so - // use the name for the guid - if (!strcmp(guid, "")) - { - strcpy(guid, name); - } -#endif // WEBRTC_LINUX mPlayoutDeviceList.emplace_back(name, guid); } @@ -543,15 +533,6 @@ void LLWebRTCImpl::updateDevices() char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; mTuningDeviceModule->RecordingDeviceName(index, name, guid); -#if WEBRTC_LINUX - // Linux audio implementation (pulse and alsa) - // return empty strings for the guid, so - // use the name for the guid - if (!strcmp(guid, "")) - { - strcpy(guid, name); - } -#endif // WEBRTC_LINUX mRecordingDeviceList.emplace_back(name, guid); } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 8830799cde..ac71e0c744 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -78,7 +78,12 @@ class LLWebRTCVoiceDevice LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id) : mDisplayName(display_name), mID(id) - {}; + { + if (mID.empty()) + { + mID = display_name; + } + }; }; typedef std::vector LLWebRTCVoiceDeviceList; diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp index f01fde08d9..9c8fcd3283 100644 --- a/indra/newview/llimview.cpp +++ b/indra/newview/llimview.cpp @@ -3005,42 +3005,42 @@ void LLIncomingCallDialog::processCallResponse(S32 response, const LLSD &payload { if (type == IM_SESSION_P2P_INVITE) { - // decline p2p voice, either via the vivox-style call mechanism + // decline p2p voice, either via the vivox-style call mechanism // or via the webrtc-style "decline p2p" mechanism. - LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(payload["voice_channel_info"]); - if (call) - { - call->declineInvite(); - } + LLVoiceP2PIncomingCallInterfacePtr call = LLVoiceClient::getInstance()->getIncomingCallInterface(payload["voice_channel_info"]); + if (call) + { + call->declineInvite(); + } else { // webrtc-style decline. - LLViewerRegion *region = gAgent.getRegion(); - if (region) - { - std::string url = region->getCapability("ChatSessionRequest"); + LLViewerRegion *region = gAgent.getRegion(); + if (region) + { + std::string url = region->getCapability("ChatSessionRequest"); - LLSD data; - data["method"] = "decline p2p voice"; - data["session-id"] = session_id; + LLSD data; + data["method"] = "decline p2p voice"; + data["session-id"] = session_id; - LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "P2P declined", "P2P decline failed."); - } + LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "P2P declined", "P2P decline failed."); + } } } else { - LLViewerRegion *region = gAgent.getRegion(); - if (region) - { - std::string url = region->getCapability("ChatSessionRequest"); + LLViewerRegion *region = gAgent.getRegion(); + if (region) + { + std::string url = region->getCapability("ChatSessionRequest"); - LLSD data; - data["method"] = "decline invitation"; - data["session-id"] = session_id; + LLSD data; + data["method"] = "decline invitation"; + data["session-id"] = session_id; - LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "Invitation declined", "Invitation decline failed."); - } + LLCoreHttpUtil::HttpCoroutineAdapter::messageHttpPost(url, data, "Invitation declined", "Invitation decline failed."); + } } } From 39cb0bca32b4017e0d0c0ef7dc4433b97f4bc1ee Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Mon, 8 Apr 2024 21:29:12 -0700 Subject: [PATCH 21/21] Show mute/unmute status in group voice --- indra/newview/llvoicewebrtc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index cb01a079cc..710067c2bf 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -2823,6 +2823,11 @@ void LLVoiceWebRTCConnection::OnDataReceivedImpl(const std::string &data, bool b { participant->mIsSpeaking = voice_data[participant_id].get("v", Json::Value(false)).asBool(); } + + if (voice_data[participant_id].isMember("m")) + { + participant->mIsModeratorMuted = voice_data[participant_id].get("m", Json::Value(false)).asBool(); + } } } }