livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

code beautification/comments

master
Roxie Linden 2024-03-09 23:00:00 -08:00
parent c7efdefb2c
commit 7714850fbe
5 changed files with 333 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
indra/llwebrtc/llwebrtc.cpp
View File

@ -44,12 +44,16 @@ LLAudioDeviceObserver::LLAudioDeviceObserver() : mSumVector {0}, mMicrophoneEner
float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; }
// TODO: Pull smoothing/filtering code into a common helper function
// for LLAudioDeviceObserver and LLCustomProcessor
void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec)
{
// calculate the energy
float energy = 0;
const short *samples = (const short *) audio_samples;
for (size_t index = 0; index < num_samples * num_channels; index++)
@ -82,7 +86,8 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples,
LLCustomProcessor::LLCustomProcessor() :
mSampleRateHz(0),
mNumChannels(0)
mNumChannels(0),
mMicrophoneEnergy(0.0)
{
memset(mSumVector, 0, sizeof(mSumVector));
}
@ -107,6 +112,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
return;
}
// grab the input audio
frame_samples.resize(stream_config.num_samples());
frame.resize(stream_config.num_channels());
for (size_t ch = 0; ch < stream_config.num_channels(); ++ch)
@ -116,6 +122,7 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
audio_in->CopyTo(stream_config, &frame[0]);
// calculate the energy
float energy = 0;
for (size_t index = 0; index < stream_config.num_samples(); index++)
{
@ -137,17 +144,33 @@ void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size));
}
//
// LLWebRTCImpl implementation
//
LLWebRTCImpl::LLWebRTCImpl() :
mPeerCustomProcessor(nullptr),
mMute(true),
mPlayoutDevice(0),
mRecordingDevice(0),
mTuningAudioDeviceObserver(nullptr)
{
}
void LLWebRTCImpl::init()
{
RTC_DCHECK(mPeerConnectionFactory);
mPlayoutDevice = 0;
mRecordingDevice = 0;
rtc::InitializeSSL();
// Normal logging is rather spammy, so turn it off.
rtc::LogMessage::LogToDebug(rtc::LS_NONE);
rtc::LogMessage::SetLogToStderr(true);
mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory();
// Create the native threads.
mNetworkThread = rtc::Thread::CreateWithSocketServer();
mNetworkThread->SetName("WebRTCNetworkThread", nullptr);
mNetworkThread->Start();
@ -162,9 +185,12 @@ void LLWebRTCImpl::init()
mWorkerThread->PostTask(
[this]()
{
// Initialize the audio devices on the Worker Thread
mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(
webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, mTaskQueueFactory.get(),
std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(),
std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
mTuningDeviceModule->Init();
mTuningDeviceModule->SetStereoRecording(true);
mTuningDeviceModule->SetStereoPlayout(true);
@ -176,9 +202,13 @@ void LLWebRTCImpl::init()
mWorkerThread->BlockingCall(
[this]()
{
// the peer device module doesn't need an observer
// as we pull peer data after audio processing.
mPeerDeviceModule =
webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(), nullptr);
webrtc::CreateAudioDeviceWithDataObserver(
webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(),
nullptr);
mPeerDeviceModule->Init();
mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice);
mPeerDeviceModule->SetRecordingDevice(mRecordingDevice);
@ -191,11 +221,15 @@ void LLWebRTCImpl::init()
mPeerDeviceModule->InitPlayout();
});
// The custom processor allows us to retrieve audio data (and levels)
// from after other audio processing such as AEC, AGC, etc.
mPeerCustomProcessor = new LLCustomProcessor;
webrtc::AudioProcessingBuilder apb;
apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mPeerCustomProcessor));
rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create();
// TODO: wire some of these to the primary interface and ultimately
// to the UI to allow user config.
webrtc::AudioProcessing::Config apm_config;
apm_config.echo_canceller.enabled = true;
apm_config.echo_canceller.mobile_mode = false;
@ -242,6 +276,41 @@ void LLWebRTCImpl::init()
});
}
void LLWebRTCImpl::terminate()
{
for (auto &connection : mPeerConnections)
{
connection->terminate();
}
mPeerConnections.clear();
mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; });
mWorkerThread->BlockingCall(
[this]()
{
if (mTuningDeviceModule)
{
mTuningDeviceModule->StopRecording();
mTuningDeviceModule->Terminate();
}
if (mPeerDeviceModule)
{
mPeerDeviceModule->StopRecording();
mPeerDeviceModule->Terminate();
}
mTuningDeviceModule = nullptr;
mPeerDeviceModule = nullptr;
mTaskQueueFactory = nullptr;
});
}
//
// Devices functions
//
// Most device-related functionality needs to happen
// on the worker thread (the audio thread,) so those calls will be
// proxied over to that thread.
//
void LLWebRTCImpl::setRecording(bool recording)
{
mWorkerThread->PostTask(
@ -258,38 +327,6 @@ void LLWebRTCImpl::setRecording(bool recording)
});
}
void LLWebRTCImpl::terminate()
{
for (auto& connection : mPeerConnections)
{
connection->terminate();
}
mPeerConnections.clear();
mSignalingThread->BlockingCall(
[this]()
{
mPeerConnectionFactory = nullptr;
});
mWorkerThread->BlockingCall(
[this]()
{
if (mTuningDeviceModule)
{
mTuningDeviceModule->StopRecording();
mTuningDeviceModule->Terminate();
}
if (mPeerDeviceModule)
{
mPeerDeviceModule->StopRecording();
mPeerDeviceModule->Terminate();
}
mTuningDeviceModule = nullptr;
mPeerDeviceModule = nullptr;
mTaskQueueFactory = nullptr;
});
}
void LLWebRTCImpl::refreshDevices()
{
mWorkerThread->PostTask([this]() { updateDevices(); });
@ -307,6 +344,8 @@ void LLWebRTCImpl::unsetDevicesObserver(LLWebRTCDevicesObserver *observer)
}
}
// TODO: There's potential for shared code here as the patterns
// are similar.
void LLWebRTCImpl::setCaptureDevice(const std::string &id)
{
mWorkerThread->PostTask(
@ -377,7 +416,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
mTuningDeviceModule->PlayoutDeviceName(i, name, guid);
if (id == guid || id == "Default")
{
RTC_LOG(LS_INFO) << __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i;
RTC_LOG(LS_INFO) << __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i;
tuningPlayoutDevice = i;
break;
}
@ -407,7 +446,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
if (id == guid || id == "Default")
{
RTC_LOG(LS_INFO)
<< __FUNCTION__ << "Set recording device to " << name << " " << guid << " " << i;
<< __FUNCTION__ << "Set playout device to " << name << " " << guid << " " << i;
mPlayoutDevice = i;
break;
}
@ -422,6 +461,7 @@ void LLWebRTCImpl::setRenderDevice(const std::string &id)
});
}
// updateDevices needs to happen on the worker thread.
void LLWebRTCImpl::updateDevices()
{
int16_t renderDeviceCount = mTuningDeviceModule->PlayoutDevices();
@ -482,13 +522,14 @@ void LLWebRTCImpl::setTuningMode(bool enable)
float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); }
float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); }
float LLWebRTCImpl::getPeerConnectionAudioLevel() { return -20 * log10f(mPeerCustomProcessor->getMicrophoneEnergy()); }
//
// Helpers
// Peer Connection Helpers
//
LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection()
LLWebRTCPeerConnectionInterface *LLWebRTCImpl::newPeerConnection()
{
rtc::scoped_refptr<LLWebRTCPeerConnectionImpl> peerConnection = rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>(new rtc::RefCountedObject<LLWebRTCPeerConnectionImpl>());
peerConnection->init(this);
@ -498,7 +539,7 @@ LLWebRTCPeerConnection * LLWebRTCImpl::newPeerConnection()
return peerConnection.get();
}
void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection)
void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection)
{
std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>>::iterator it =
std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection);
@ -513,6 +554,20 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnection * peer_connection)
}
}
//
// LLWebRTCPeerConnectionImpl implementation.
//
// Most peer connection (signaling) happens on
// the signaling thread.
LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() :
mWebRTCImpl(nullptr),
mMute(false),
mAnswerReceived(false)
{
}
//
// LLWebRTCPeerConnection interface
//
@ -547,6 +602,9 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve
}
}
// TODO: Add initialization structure through which
// stun and turn servers may be passed in from
// the sim or login.
bool LLWebRTCPeerConnectionImpl::initializeConnection()
{
RTC_DCHECK(!mPeerConnection);
@ -663,7 +721,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection()
}
for (auto &observer : mSignalingObserverList)
{
observer->OnPeerShutDown();
observer->OnPeerConnectionShutdown();
}
});
return true;
@ -673,7 +731,7 @@ bool LLWebRTCPeerConnectionImpl::shutdownConnection()
void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable)
{
// set_enabled shouldn't be done on the worker thread
// set_enabled shouldn't be done on the worker thread.
if (mPeerConnection)
{
auto senders = mPeerConnection->GetSenders();
@ -697,6 +755,7 @@ void LLWebRTCPeerConnectionImpl::enableReceiverTracks(bool enable)
}
}
// Tell the peer connection that we've received a SDP answer from the sim.
void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
{
RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp;
@ -713,6 +772,11 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
});
}
//
// LLWebRTCAudioInterface implementation
//
void LLWebRTCPeerConnectionImpl::setMute(bool mute)
{
mMute = mute;
@ -812,21 +876,21 @@ void LLWebRTCPeerConnectionImpl::OnDataChannel(rtc::scoped_refptr<webrtc::DataCh
void LLWebRTCPeerConnectionImpl::OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState new_state)
{
LLWebRTCSignalingObserver::IceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
LLWebRTCSignalingObserver::EIceGatheringState webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
switch (new_state)
{
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringNew:
webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
break;
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringGathering:
webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_GATHERING;
webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_GATHERING;
break;
case webrtc::PeerConnectionInterface::IceGatheringState::kIceGatheringComplete:
webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE;
webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE;
break;
default:
RTC_LOG(LS_ERROR) << __FUNCTION__ << " Bad Ice Gathering State" << new_state;
webrtc_new_state = LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW;
webrtc_new_state = LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW;
return;
}
@ -875,6 +939,8 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf
}
}
// Convert an ICE candidate into a string appropriate for trickling
// to the Secondlife WebRTC server via the sim.
static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterface *candidate)
{
std::ostringstream candidate_stream;
@ -920,6 +986,7 @@ static std::string iceCandidateToTrickleString(const webrtc::IceCandidateInterfa
return candidate_stream.str();
}
// The webrtc library has a new ice candidate.
void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterface *candidate)
{
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
@ -931,19 +998,24 @@ void LLWebRTCPeerConnectionImpl::OnIceCandidate(const webrtc::IceCandidateInterf
}
if (mAnswerReceived)
{
// We've already received an answer SDP from the Secondlife WebRTC server
// so simply tell observers about our new ice candidate.
for (auto &observer : mSignalingObserverList)
{
LLWebRTCIceCandidate ice_candidate;
ice_candidate.candidate = iceCandidateToTrickleString(candidate);
ice_candidate.mline_index = candidate->sdp_mline_index();
ice_candidate.sdp_mid = candidate->sdp_mid();
ice_candidate.mCandidate = iceCandidateToTrickleString(candidate);
ice_candidate.mMLineIndex = candidate->sdp_mline_index();
ice_candidate.mSdpMid = candidate->sdp_mid();
observer->OnIceCandidate(ice_candidate);
}
}
else
{
// As we've not yet received our answer, cache the candidate.
mCachedIceCandidates.push_back(
webrtc::CreateIceCandidate(candidate->sdp_mid(), candidate->sdp_mline_index(), candidate->candidate()));
webrtc::CreateIceCandidate(candidate->sdp_mid(),
candidate->sdp_mline_index(),
candidate->candidate()));
}
}
@ -994,13 +1066,18 @@ void LLWebRTCPeerConnectionImpl::OnSuccess(webrtc::SessionDescriptionInterface *
rtc::scoped_refptr<webrtc::SetLocalDescriptionObserverInterface>(this));
}
void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error) { RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message(); }
void LLWebRTCPeerConnectionImpl::OnFailure(webrtc::RTCError error)
{
RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message();
}
//
// SetRemoteDescriptionObserverInterface implementation.
//
void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError error)
{
// we've received an answer SDP from the sim.
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << mPeerConnection->signaling_state();
if (!error.ok())
{
@ -1008,14 +1085,16 @@ void LLWebRTCPeerConnectionImpl::OnSetRemoteDescriptionComplete(webrtc::RTCError
return;
}
mAnswerReceived = true;
// tell the observers about any cached ICE candidates.
for (auto &observer : mSignalingObserverList)
{
for (auto &candidate : mCachedIceCandidates)
{
LLWebRTCIceCandidate ice_candidate;
ice_candidate.candidate = iceCandidateToTrickleString(candidate.get());
ice_candidate.mline_index = candidate->sdp_mline_index();
ice_candidate.sdp_mid = candidate->sdp_mid();
ice_candidate.mCandidate = iceCandidateToTrickleString(candidate.get());
ice_candidate.mMLineIndex = candidate->sdp_mline_index();
ice_candidate.mSdpMid = candidate->sdp_mid();
observer->OnIceCandidate(ice_candidate);
}
}
@ -1061,7 +1140,6 @@ void LLWebRTCPeerConnectionImpl::OnStateChange()
}
}
void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer)
{
std::string data((const char*)buffer.data.cdata(), buffer.size());
@ -1071,6 +1149,10 @@ void LLWebRTCPeerConnectionImpl::OnMessage(const webrtc::DataBuffer& buffer)
}
}
//
// LLWebRTCDataInterface
//
void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)
{
if (mDataChannel)
@ -1081,7 +1163,10 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)
}
}
void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer) { mDataObserverList.emplace_back(observer); }
void LLWebRTCPeerConnectionImpl::setDataObserver(LLWebRTCDataObserver* observer)
{
mDataObserverList.emplace_back(observer);
}
void LLWebRTCPeerConnectionImpl::unsetDataObserver(LLWebRTCDataObserver* observer)
{
@ -1099,12 +1184,12 @@ LLWebRTCDeviceInterface * getDeviceInterface()
return gWebRTCImpl;
}
LLWebRTCPeerConnection * newPeerConnection()
LLWebRTCPeerConnectionInterface* newPeerConnection()
{
return gWebRTCImpl->newPeerConnection();
}
void freePeerConnection(LLWebRTCPeerConnection *peer_connection)
void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection)
{
gWebRTCImpl->freePeerConnection(peer_connection);
}
@ -1112,7 +1197,7 @@ void freePeerConnection(LLWebRTCPeerConnection *peer_connection)
void init()
{
gWebRTCImpl = new LLWebRTCImpl();
gWebRTCImpl = new LLWebRTCImpl();
gWebRTCImpl->init();
}

129
indra/llwebrtc/llwebrtc.h
View File

@ -24,6 +24,17 @@
* $/LicenseInfo$
*/
/*
* llwebrtc wraps the native webrtc c++ library in a dynamic library with a simlified interface
* so that the viewer can use it. This is done because native webrtc has a different
* overall threading model than the viewer.
* The native webrtc library is also compiled with clang, and has memory management
* functions that conflict namespace-wise with those in the viewer.
*
* Due to these differences, code from the viewer cannot be pulled in to this
* dynamic library, so it remains very simple.
*/
#ifndef LLWEBRTC_H
#define LLWEBRTC_H
@ -45,52 +56,74 @@
namespace llwebrtc
{
struct LLWebRTCIceCandidate
{
std::string candidate;
std::string sdp_mid;
int mline_index;
};
// LLWebRTCVoiceDevice is a simple representation of the
// components of a device, used to communicate this
// information to the viewer.
// A note on threading.
// Native WebRTC has it's own threading model. Some discussion
// can be found here (https://webrtc.github.io/webrtc-org/native-code/native-apis/)
//
// Note that all callbacks to observers will occurr on one of the WebRTC native threads
// (signaling, worker, etc.) Care should be taken to assure there are not
// bad interactions with the viewer threads.
class LLWebRTCVoiceDevice
{
public:
std::string display_name; // friendly value for the user
std::string id; // internal value for selection
bool current; // current device
std::string mDisplayName; // friendly name for user interface purposes
std::string mID; // internal value for selection
bool mCurrent; // current device
LLWebRTCVoiceDevice(const std::string &display_name, const std::string &id, bool current) :
display_name(display_name),
id(id),
current(current)
mDisplayName(display_name),
mID(id),
mCurrent(current)
{};
};
typedef std::vector<LLWebRTCVoiceDevice> LLWebRTCVoiceDeviceList;
// The LLWebRTCDeviceObserver should be implemented by the viewer
// webrtc module, which will receive notifications when devices
// change (are unplugged, etc.)
class LLWebRTCDevicesObserver
{
public:
virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices, const LLWebRTCVoiceDeviceList &capture_devices) = 0;
virtual void OnDevicesChanged(const LLWebRTCVoiceDeviceList &render_devices,
const LLWebRTCVoiceDeviceList &capture_devices) = 0;
};
// The LLWebRTCDeviceInterface provides a way for the viewer
// to enumerate, set, and get notifications of changes
// for both capture (microphone) and render (speaker)
// devices.
class LLWebRTCDeviceInterface
{
public:
// instructs webrtc to refresh the device list.
virtual void refreshDevices() = 0;
// set the capture and render devices using the unique identifier for the device
virtual void setCaptureDevice(const std::string& id) = 0;
virtual void setRenderDevice(const std::string& id) = 0;
// Device observers for device change callbacks.
virtual void setDevicesObserver(LLWebRTCDevicesObserver *observer) = 0;
virtual void unsetDevicesObserver(LLWebRTCDevicesObserver *observer) = 0;
// tuning and audio levels
virtual void setTuningMode(bool enable) = 0;
virtual float getTuningAudioLevel() = 0;
virtual float getPeerAudioLevel() = 0;
virtual float getTuningAudioLevel() = 0; // for use during tuning
virtual float getPeerConnectionAudioLevel() = 0; // for use when not tuning
};
// LLWebRTCAudioInterface provides the viewer with a way
// to set audio characteristics (mute, send and receive volume)
class LLWebRTCAudioInterface
{
public:
@ -99,41 +132,81 @@ class LLWebRTCAudioInterface
virtual void setSendVolume(float volume) = 0; // volume between 0.0 and 1.0
};
// LLWebRTCDataObserver allows the viewer voice module to be notified when
// data is received over the data channel.
class LLWebRTCDataObserver
{
public:
virtual void OnDataReceived(const std::string& data, bool binary) = 0;
};
// LLWebRTCDataInterface allows the viewer to send data over the data channel.
class LLWebRTCDataInterface
{
public:
virtual void sendData(const std::string& data, bool binary=false) = 0;
virtual void setDataObserver(LLWebRTCDataObserver *observer) = 0;
virtual void unsetDataObserver(LLWebRTCDataObserver *observer) = 0;
};
// LLWebRTCIceCandidate is a basic structure containing
// information needed for ICE trickling.
struct LLWebRTCIceCandidate
{
std::string mCandidate;
std::string mSdpMid;
int mMLineIndex;
};
// LLWebRTCSignalingObserver provides a way for the native
// webrtc library to notify the viewer voice module of
// various state changes.
class LLWebRTCSignalingObserver
{
public:
enum IceGatheringState{
public:
typedef enum e_ice_gathering_state {
ICE_GATHERING_NEW,
ICE_GATHERING_GATHERING,
ICE_GATHERING_COMPLETE
};
virtual void OnIceGatheringState(IceGatheringState state) = 0;
} EIceGatheringState;
// Called when ICE gathering states have changed.
// This may be called at any time, as ICE gathering
// can be redone while a connection is up.
virtual void OnIceGatheringState(EIceGatheringState state) = 0;
// Called when a new ice candidate is available.
virtual void OnIceCandidate(const LLWebRTCIceCandidate& candidate) = 0;
// Called when an offer is available after a connection is requested.
virtual void OnOfferAvailable(const std::string& sdp) = 0;
// Called when a connection enters a failure state and renegotiation is needed.
virtual void OnRenegotiationNeeded() = 0;
// Called when the audio channel has been established and audio
// can begin.
virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0;
// Called when the data channel has been established and data
// transfer can begin.
virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0;
virtual void OnPeerShutDown() = 0;
// Called when a peer connection has finished shutting down.
virtual void OnPeerConnectionShutdown() = 0;
};
class LLWebRTCPeerConnection
// LLWebRTCPeerConnectionInterface representsd a connection to a peer,
// in most cases a Secondlife WebRTC server. This interface
// allows for management of this peer connection.
class LLWebRTCPeerConnectionInterface
{
public:
virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
@ -142,11 +215,21 @@ class LLWebRTCPeerConnection
virtual void AnswerAvailable(const std::string &sdp) = 0;
};
// The following define the dynamic linked library
// exports.
// This library must be initialized before use.
LLSYMEXPORT void init();
// And should be terminated as part of shutdown.
LLSYMEXPORT void terminate();
// Return an interface for device management.
LLSYMEXPORT LLWebRTCDeviceInterface* getDeviceInterface();
LLSYMEXPORT LLWebRTCPeerConnection* newPeerConnection();
LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnection *connection);
// Allocate and free peer connections.
LLSYMEXPORT LLWebRTCPeerConnectionInterface* newPeerConnection();
LLSYMEXPORT void freePeerConnection(LLWebRTCPeerConnectionInterface *connection);
}
#endif // LLWEBRTC_H

68
indra/llwebrtc/llwebrtc_impl.h
View File

@ -1,6 +1,6 @@
/**
* @file llwebrtc_impl.h
* @brief WebRTC interface implementation header
* @brief WebRTC dynamic library implementation header
*
* $LicenseInfo:firstyear=2023&license=viewerlgpl$
* Second Life Viewer Source Code
@ -65,19 +65,27 @@ namespace llwebrtc
class LLWebRTCPeerConnectionImpl;
// Implements a class allowing capture of audio data
// to determine audio level of the microphone.
class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
{
public:
LLAudioDeviceObserver();
// Retrieve the RMS audio loudness
float getMicrophoneEnergy();
// Data retrieved from the caputure device is
// passed in here for processing.
void OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec) override;
// This is for data destined for the render device.
// not currently used.
void OnRenderData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
@ -85,10 +93,13 @@ class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
const uint32_t samples_per_sec) override;
protected:
float mSumVector[30]; // 300 ms of smoothing
static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing (30 frames)
float mSumVector[NUM_PACKETS_TO_FILTER];
float mMicrophoneEnergy;
};
// Used to process/retrieve audio levels after
// all of the processing (AGC, AEC, etc.) for display in-world to the user.
class LLCustomProcessor : public webrtc::CustomProcessing
{
public:
@ -97,8 +108,10 @@ class LLCustomProcessor : public webrtc::CustomProcessing
// (Re-) Initializes the submodule.
void Initialize(int sample_rate_hz, int num_channels) override;
// Analyzes the given capture or render signal.
void Process(webrtc::AudioBuffer *audio) override;
// Returns a string representation of the module state.
std::string ToString() const override { return ""; }
@ -113,13 +126,13 @@ class LLCustomProcessor : public webrtc::CustomProcessing
float mMicrophoneEnergy;
};
// Primary singleton implementation for interfacing
// with the native webrtc library.
class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceSink
{
public:
LLWebRTCImpl() :
mPeerCustomProcessor(nullptr), mMute(true)
{
}
LLWebRTCImpl();
~LLWebRTCImpl() {}
void init();
@ -139,7 +152,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
void setTuningMode(bool enable) override;
float getTuningAudioLevel() override;
float getPeerAudioLevel() override;
float getPeerConnectionAudioLevel() override;
//
// AudioDeviceSink
@ -150,6 +163,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
// Helpers
//
// The following thread helpers allow the
// LLWebRTCPeerConnectionImpl class to post
// tasks to the native webrtc threads.
void PostWorkerTask(absl::AnyInvocable<void() &&> task,
const webrtc::Location& location = webrtc::Location::Current())
{
@ -185,21 +201,31 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
{
mNetworkThread->BlockingCall(std::move(functor), location);
}
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory() { return mPeerConnectionFactory; }
LLWebRTCPeerConnection * newPeerConnection();
void freePeerConnection(LLWebRTCPeerConnection * peer_connection);
// Allows the LLWebRTCPeerConnectionImpl class to retrieve the
// native webrtc PeerConnectionFactory.
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> getPeerConnectionFactory()
{
return mPeerConnectionFactory;
}
// create or destroy a peer connection.
LLWebRTCPeerConnectionInterface* newPeerConnection();
void freePeerConnection(LLWebRTCPeerConnectionInterface* peer_connection);
// enables/disables capture via the capture device
void setRecording(bool recording);
protected:
// The native webrtc threads
std::unique_ptr<rtc::Thread> mNetworkThread;
std::unique_ptr<rtc::Thread> mWorkerThread;
std::unique_ptr<rtc::Thread> mSignalingThread;
// The factory that allows creation of native webrtc PeerConnections.
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory;
webrtc::PeerConnectionInterface::RTCConfiguration mConfiguration;
// more native webrtc stuff
std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory;
@ -209,11 +235,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
rtc::scoped_refptr<webrtc::AudioDeviceModule> mPeerDeviceModule;
std::vector<LLWebRTCDevicesObserver *> mVoiceDevicesObserverList;
// accessors in webrtc aren't apparently implemented yet.
// accessors in native webrtc for devices aren't apparently implemented yet.
int32_t mPlayoutDevice;
int32_t mRecordingDevice;
bool mMute;
LLAudioDeviceObserver * mTuningAudioDeviceObserver;
LLCustomProcessor * mPeerCustomProcessor;
@ -221,7 +247,11 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>> mPeerConnections;
};
class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
// The implementation of a peer connection, which contains
// the various interfaces used by the viewer to interact with
// the webrtc connection.
class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface,
public LLWebRTCAudioInterface,
public LLWebRTCDataInterface,
public webrtc::PeerConnectionObserver,
@ -232,7 +262,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
{
public:
LLWebRTCPeerConnectionImpl() {}
LLWebRTCPeerConnectionImpl();
~LLWebRTCPeerConnectionImpl() {}
void init(LLWebRTCImpl * webrtc_impl);
@ -270,7 +300,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
//
void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState new_state) override {}
void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
void OnAddTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> &streams) override;
void OnRemoveTrack(rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) override;
void OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel) override;
@ -311,6 +341,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
protected:
LLWebRTCImpl * mWebRTCImpl;
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory;
bool mMute;
@ -323,6 +354,7 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnection,
rtc::scoped_refptr<webrtc::PeerConnectionInterface> mPeerConnection;
rtc::scoped_refptr<webrtc::MediaStreamInterface> mLocalStream;
// data
std::vector<LLWebRTCDataObserver *> mDataObserverList;
rtc::scoped_refptr<webrtc::DataChannelInterface> mDataChannel;
};

50
indra/newview/llvoicewebrtc.cpp
View File

@ -597,8 +597,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi
bool renderDeviceSet = false;
for (auto &device : render_devices)
{
addRenderDevice(LLVoiceDevice(device.display_name, device.id));
if (device.current && outputDevice == device.id)
addRenderDevice(LLVoiceDevice(device.mDisplayName, device.mID));
if (device.mCurrent && outputDevice == device.mID)
{
setRenderDevice(outputDevice);
renderDeviceSet = true;
@ -613,8 +613,8 @@ void LLWebRTCVoiceClient::OnDevicesChanged(const llwebrtc::LLWebRTCVoiceDeviceLi
bool captureDeviceSet = false;
for (auto &device : capture_devices)
{
addCaptureDevice(LLVoiceDevice(device.display_name, device.id));
if (device.current && inputDevice == device.id)
addCaptureDevice(LLVoiceDevice(device.mDisplayName, device.mID));
if (device.mCurrent && inputDevice == device.mID)
{
setCaptureDevice(outputDevice);
captureDeviceSet = true;
@ -696,7 +696,7 @@ float LLWebRTCVoiceClient::getAudioLevel()
}
else
{
return (1.0 - mWebRTCDeviceInterface->getPeerAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1;
return (1.0 - mWebRTCDeviceInterface->getPeerConnectionAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1;
}
}
@ -2030,8 +2030,8 @@ LLVoiceWebRTCConnection::LLVoiceWebRTCConnection(const LLUUID &regionID, const s
mChannelID(channelID),
mRegionID(regionID)
{
mWebRTCPeerConnection = llwebrtc::newPeerConnection();
mWebRTCPeerConnection->setSignalingObserver(this);
mWebRTCPeerConnectionInterface = llwebrtc::newPeerConnection();
mWebRTCPeerConnectionInterface->setSignalingObserver(this);
}
LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
@ -2042,10 +2042,10 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
// by llwebrtc::terminate() on shutdown.
return;
}
if (mWebRTCPeerConnection)
if (mWebRTCPeerConnectionInterface)
{
llwebrtc::freePeerConnection(mWebRTCPeerConnection);
mWebRTCPeerConnection = nullptr;
llwebrtc::freePeerConnection(mWebRTCPeerConnectionInterface);
mWebRTCPeerConnectionInterface = nullptr;
}
}
@ -2058,19 +2058,19 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection()
// negotiated, updates about the best connectivity paths may trickle in. These need to be
// sent to the Secondlife WebRTC server via the simulator so that both sides have a clear
// view of the network environment.
void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::IceGatheringState state)
void LLVoiceWebRTCConnection::OnIceGatheringState(llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState state)
{
LL_DEBUGS("Voice") << "Ice Gathering voice account. " << state << LL_ENDL;
switch (state)
{
case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_COMPLETE:
case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_COMPLETE:
{
LLMutexLock lock(&mVoiceStateMutex);
mIceCompleted = true;
break;
}
case llwebrtc::LLWebRTCSignalingObserver::IceGatheringState::ICE_GATHERING_NEW:
case llwebrtc::LLWebRTCSignalingObserver::EIceGatheringState::ICE_GATHERING_NEW:
{
LLMutexLock lock(&mVoiceStateMutex);
mIceCompleted = false;
@ -2167,9 +2167,9 @@ void LLVoiceWebRTCConnection::processIceUpdates()
for (auto &ice_candidate : mIceCandidates)
{
LLSD body_candidate;
body_candidate["sdpMid"] = ice_candidate.sdp_mid;
body_candidate["sdpMLineIndex"] = ice_candidate.mline_index;
body_candidate["candidate"] = ice_candidate.candidate;
body_candidate["sdpMid"] = ice_candidate.mSdpMid;
body_candidate["sdpMLineIndex"] = ice_candidate.mMLineIndex;
body_candidate["candidate"] = ice_candidate.mCandidate;
candidates.append(body_candidate);
}
body["candidates"] = candidates;
@ -2242,7 +2242,7 @@ void LLVoiceWebRTCConnection::OnRenegotiationNeeded()
}
}
void LLVoiceWebRTCConnection::OnPeerShutDown()
void LLVoiceWebRTCConnection::OnPeerConnectionShutdown()
{
setVoiceConnectionState(VOICE_STATE_SESSION_EXIT);
mOutstandingRequests--; // shut down is an async call which is handled on a webrtc thread.
@ -2364,9 +2364,9 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess(const LLSD &res
return;
}
if (mWebRTCPeerConnection)
if (mWebRTCPeerConnectionInterface)
{
if (mWebRTCPeerConnection->shutdownConnection())
if (mWebRTCPeerConnectionInterface->shutdownConnection())
{
mOutstandingRequests++;
}
@ -2395,10 +2395,10 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure(std::string url
boost::bind(&LLVoiceWebRTCSpatialConnection::OnVoiceDisconnectionRequestFailure, this, url, retries - 1, body, _1));
return;
}
if (mWebRTCPeerConnection)
if (mWebRTCPeerConnectionInterface)
{
mOutstandingRequests++;
mWebRTCPeerConnection->shutdownConnection();
mWebRTCPeerConnectionInterface->shutdownConnection();
}
else
{
@ -2482,7 +2482,7 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result
LL_DEBUGS("Voice") << "ProvisionVoiceAccountRequest response"
<< " channel sdp " << mRemoteChannelSDP << LL_ENDL;
mWebRTCPeerConnection->AnswerAvailable(mRemoteChannelSDP);
mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP);
}
@ -2531,7 +2531,7 @@ bool LLVoiceWebRTCConnection::connectionStateMachine()
// tell the webrtc library that we want a connection. The library will
// respond with an offer on a separate thread, which will cause
// the session state to change.
if (!mWebRTCPeerConnection->initializeConnection())
if (!mWebRTCPeerConnectionInterface->initializeConnection())
{
setVoiceConnectionState(VOICE_STATE_SESSION_RETRY);
}
@ -2830,7 +2830,7 @@ LLVoiceWebRTCSpatialConnection::~LLVoiceWebRTCSpatialConnection()
return;
}
assert(mOutstandingRequests == 0);
mWebRTCPeerConnection->unsetSignalingObserver(this);
mWebRTCPeerConnectionInterface->unsetSignalingObserver(this);
}
void LLVoiceWebRTCSpatialConnection::setMuteMic(bool muted)
@ -2873,7 +2873,7 @@ LLVoiceWebRTCAdHocConnection::~LLVoiceWebRTCAdHocConnection()
return;
}
assert(mOutstandingRequests == 0);
mWebRTCPeerConnection->unsetSignalingObserver(this);
mWebRTCPeerConnectionInterface->unsetSignalingObserver(this);
}
// Add-hoc connections require a different channel type

6
indra/newview/llvoicewebrtc.h
View File

@ -584,12 +584,12 @@ class LLVoiceWebRTCConnection :
/// @name Signaling notification
// LLWebRTCSignalingObserver
//@{
void OnIceGatheringState(IceGatheringState state) override;
void OnIceGatheringState(EIceGatheringState state) override;
void OnIceCandidate(const llwebrtc::LLWebRTCIceCandidate &candidate) override;
void OnOfferAvailable(const std::string &sdp) override;
void OnRenegotiationNeeded() override;
void OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) override;
void OnPeerShutDown() override;
void OnPeerConnectionShutdown() override;
//@}
/////////////////////////
@ -702,7 +702,7 @@ class LLVoiceWebRTCConnection :
bool mIceCompleted;
bool mTrickling;
llwebrtc::LLWebRTCPeerConnection *mWebRTCPeerConnection;
llwebrtc::LLWebRTCPeerConnectionInterface *mWebRTCPeerConnectionInterface;
llwebrtc::LLWebRTCAudioInterface *mWebRTCAudioInterface;
llwebrtc::LLWebRTCDataInterface *mWebRTCDataInterface;
};