livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add UI for managing echo cancellation, AGC, and noise control. 

Plumb audio settings through from webrtc to the sound preferences
UI (still needs some tweaking, of course.)

Also, choose stun servers based on grid.  Ultimately, the stun
stun servers will be passed up via login or something.
master
Roxie Linden 2024-03-30 21:58:00 -07:00
parent 8d14df5984
commit cdae5ebc16
9 changed files with 255 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
indra/llwebrtc/llwebrtc.cpp
View File

@ -157,7 +157,6 @@ LLWebRTCImpl::LLWebRTCImpl() :
void LLWebRTCImpl::init()
{
RTC_DCHECK(mPeerConnectionFactory);
mPlayoutDevice = 0;
mRecordingDevice = 0;
rtc::InitializeSSL();
@ -222,12 +221,10 @@ void LLWebRTCImpl::init()
mPeerCustomProcessor = new LLCustomProcessor;
webrtc::AudioProcessingBuilder apb;
apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mPeerCustomProcessor));
rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create();
mAudioProcessingModule = apb.Create();
// TODO: wire some of these to the primary interface and ultimately
// to the UI to allow user config.
webrtc::AudioProcessing::Config apm_config;
apm_config.echo_canceller.enabled = true;
apm_config.echo_canceller.enabled = false;
apm_config.echo_canceller.mobile_mode = false;
apm_config.gain_controller1.enabled = true;
apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog;
@ -250,8 +247,8 @@ void LLWebRTCImpl::init()
processing_config.reverse_output_stream().set_num_channels(2);
processing_config.reverse_output_stream().set_sample_rate_hz(48000);
apm->Initialize(processing_config);
apm->ApplyConfig(apm_config);
mAudioProcessingModule->Initialize(processing_config);
mAudioProcessingModule->ApplyConfig(apm_config);
mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(),
mWorkerThread.get(),
@ -262,7 +259,7 @@ void LLWebRTCImpl::init()
nullptr /* video_encoder_factory */,
nullptr /* video_decoder_factory */,
nullptr /* audio_mixer */,
apm);
mAudioProcessingModule);
mWorkerThread->BlockingCall([this]() { mPeerDeviceModule->StartPlayout(); });
}
@ -318,6 +315,49 @@ void LLWebRTCImpl::setRecording(bool recording)
});
}
void LLWebRTCImpl::setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config)
{
webrtc::AudioProcessing::Config apm_config;
apm_config.echo_canceller.enabled = config.mEchoCancellation;
apm_config.echo_canceller.mobile_mode = false;
apm_config.gain_controller1.enabled = true;
apm_config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog;
apm_config.gain_controller2.enabled = true;
apm_config.high_pass_filter.enabled = true;
apm_config.transient_suppression.enabled = true;
apm_config.pipeline.multi_channel_render = true;
apm_config.pipeline.multi_channel_capture = true;
apm_config.pipeline.multi_channel_capture = true;
switch (config.mNoiseSuppressionLevel)
{
case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_NONE:
apm_config.noise_suppression.enabled = false;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow;
break;
case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_LOW:
apm_config.noise_suppression.enabled = true;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow;
break;
case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_MODERATE:
apm_config.noise_suppression.enabled = true;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate;
break;
case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_HIGH:
apm_config.noise_suppression.enabled = true;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kHigh;
break;
case LLWebRTCDeviceInterface::AudioConfig::NOISE_SUPPRESSION_LEVEL_VERY_HIGH:
apm_config.noise_suppression.enabled = true;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kVeryHigh;
break;
default:
apm_config.noise_suppression.enabled = false;
apm_config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kLow;
}
mAudioProcessingModule->ApplyConfig(apm_config);
}
void LLWebRTCImpl::refreshDevices()
{
mWorkerThread->PostTask([this]() { updateDevices(); });
@ -616,32 +656,36 @@ void LLWebRTCPeerConnectionImpl::unsetSignalingObserver(LLWebRTCSignalingObserve
}
}
// TODO: Add initialization structure through which
// stun and turn servers may be passed in from
// the sim or login.
bool LLWebRTCPeerConnectionImpl::initializeConnection()
bool LLWebRTCPeerConnectionImpl::initializeConnection(LLWebRTCPeerConnectionInterface::InitOptions options)
{
RTC_DCHECK(!mPeerConnection);
mAnswerReceived = false;
mWebRTCImpl->PostSignalingTask(
[this]()
[this, options]()
{
std::vector<LLWebRTCPeerConnectionInterface::InitOptions::IceServers> servers = options.mServers;
if(servers.empty())
{
LLWebRTCPeerConnectionInterface::InitOptions::IceServers ice_servers;
ice_servers.mUrls.push_back("stun:stun.l.google.com:19302");
ice_servers.mUrls.push_back("stun1:stun.l.google.com:19302");
ice_servers.mUrls.push_back("stun2:stun.l.google.com:19302");
ice_servers.mUrls.push_back("stun3:stun.l.google.com:19302");
ice_servers.mUrls.push_back("stun4:stun.l.google.com:19302");
}
webrtc::PeerConnectionInterface::RTCConfiguration config;
for (auto server : servers)
{
webrtc::PeerConnectionInterface::IceServer ice_server;
ice_server.urls = server.mUrls;
ice_server.username = server.mUserName;
ice_server.password = server.mPassword;
config.servers.push_back(ice_server);
}
config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
webrtc::PeerConnectionInterface::IceServer server;
server.uri = "stun:roxie-turn.staging.secondlife.io:3478";
config.servers.push_back(server);
server.uri = "stun:stun.l.google.com:19302";
config.servers.push_back(server);
server.uri = "stun:stun1.l.google.com:19302";
config.servers.push_back(server);
server.uri = "stun:stun2.l.google.com:19302";
config.servers.push_back(server);
server.uri = "stun:stun3.l.google.com:19302";
config.servers.push_back(server);
server.uri = "stun:stun4.l.google.com:19302";
config.servers.push_back(server);
config.set_min_port(60000);
config.set_max_port(60100);
@ -671,7 +715,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection()
cricket::AudioOptions audioOptions;
audioOptions.auto_gain_control = true;
audioOptions.echo_cancellation = true; // incompatible with opus stereo
audioOptions.echo_cancellation = false; // incompatible with opus stereo
audioOptions.noise_suppression = true;
mLocalStream = mPeerConnectionFactory->CreateLocalMediaStream("SLStream");

44
indra/llwebrtc/llwebrtc.h
View File

@ -99,10 +99,32 @@ class LLWebRTCDevicesObserver
// to enumerate, set, and get notifications of changes
// for both capture (microphone) and render (speaker)
// devices.
class LLWebRTCDeviceInterface
{
public:
struct AudioConfig {
bool mAGC { true };
bool mEchoCancellation { true };
// TODO: The various levels of noise suppression are configured
// on the APM which would require setting config on the APM.
// We should pipe the various values through
// later.
typedef enum {
NOISE_SUPPRESSION_LEVEL_NONE = 0,
NOISE_SUPPRESSION_LEVEL_LOW,
NOISE_SUPPRESSION_LEVEL_MODERATE,
NOISE_SUPPRESSION_LEVEL_HIGH,
NOISE_SUPPRESSION_LEVEL_VERY_HIGH
} ENoiseSuppressionLevel;
ENoiseSuppressionLevel mNoiseSuppressionLevel { NOISE_SUPPRESSION_LEVEL_VERY_HIGH };
};
virtual void setAudioConfig(AudioConfig config) = 0;
// instructs webrtc to refresh the device list.
virtual void refreshDevices() = 0;
@ -194,19 +216,35 @@ class LLWebRTCSignalingObserver
virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0;
};
// LLWebRTCPeerConnectionInterface representsd a connection to a peer,
// in most cases a Secondlife WebRTC server. This interface
// allows for management of this peer connection.
class LLWebRTCPeerConnectionInterface
{
public:
struct InitOptions
{
// equivalent of PeerConnectionInterface::IceServer
struct IceServers {
// Valid formats are described in RFC7064 and RFC7065.
// Urls should containe dns hostnames (not IP addresses)
// as the TLS certificate policy is 'secure.'
// and we do not currentply support TLS extensions.
std::vector<std::string> mUrls;
std::string mUserName;
std::string mPassword;
};
std::vector<IceServers> mServers;
};
virtual bool initializeConnection(InitOptions options = InitOptions()) = 0;
virtual bool shutdownConnection() = 0;
virtual void setSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
virtual void unsetSignalingObserver(LLWebRTCSignalingObserver* observer) = 0;
virtual bool initializeConnection() = 0;
virtual bool shutdownConnection() = 0;
virtual void AnswerAvailable(const std::string &sdp) = 0;
};

8
indra/llwebrtc/llwebrtc_impl.h
View File

@ -145,6 +145,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
// LLWebRTCDeviceInterface
//
void setAudioConfig(LLWebRTCDeviceInterface::AudioConfig config = LLWebRTCDeviceInterface::AudioConfig()) override;
void refreshDevices() override;
void setDevicesObserver(LLWebRTCDevicesObserver *observer) override;
@ -227,6 +229,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
// The factory that allows creation of native webrtc PeerConnections.
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> mPeerConnectionFactory;
rtc::scoped_refptr<webrtc::AudioProcessing> mAudioProcessingModule;
// more native webrtc stuff
std::unique_ptr<webrtc::TaskQueueFactory> mTaskQueueFactory;
@ -278,11 +282,11 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface,
//
// LLWebRTCPeerConnection
//
bool initializeConnection(InitOptions options = InitOptions()) override;
bool shutdownConnection() override;
void setSignalingObserver(LLWebRTCSignalingObserver *observer) override;
void unsetSignalingObserver(LLWebRTCSignalingObserver *observer) override;
bool initializeConnection() override;
bool shutdownConnection() override;
void AnswerAvailable(const std::string &sdp) override;
//

33
indra/newview/app_settings/settings.xml
View File

@ -15108,6 +15108,39 @@
<key>Value</key>
<integer>44125</integer>
</map>
<key>VoiceEchoCancellation</key>
<map>
<key>Comment</key>
<string>Voice Echo Cancellation</string>
<key>Persist</key>
<integer>1</integer>
<key>Type</key>
<string>Boolean</string>
<key>Value</key>
<integer>1</integer>
</map>
<key>VoiceAutomaticGainControl</key>
<map>
<key>Comment</key>
<string>Voice Automatic Gain Control</string>
<key>Persist</key>
<integer>1</integer>
<key>Type</key>
<string>Boolean</string>
<key>Value</key>
<integer>1</integer>
</map>
<key>VoiceNoiseSuppressionLevel</key>
<map>
<key>Comment</key>
<string>Voice Noise Suppression Level</string>
<key>Persist</key>
<integer>1</integer>
<key>Type</key>
<string>U32</string>
<key>Value</key>
<integer>4</integer>
</map>
<key>WarningsAsChat</key>
<map>
<key>Comment</key>

3
indra/newview/llviewercontrol.cpp
View File

@ -803,6 +803,9 @@ void settings_setup_listeners()
setting_setup_signal_listener(gSavedSettings, "PushToTalkButton", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "PushToTalkToggle", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceEarLocation", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceEchoCancellation", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceAutomaticGainControl", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceNoiseSuppressionLevel", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceInputAudioDevice", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "VoiceOutputAudioDevice", handleVoiceClientPrefsChanged);
setting_setup_signal_listener(gSavedSettings, "AudioLevelMic", handleVoiceClientPrefsChanged);

4
indra/newview/llvoicevivox.cpp
View File

@ -1154,7 +1154,9 @@ bool LLVivoxVoiceClient::provisionVoiceAccount()
do
{
LLVoiceVivoxStats::getInstance()->provisionAttemptStart();
result = httpAdapter->postAndSuspend(httpRequest, url, LLSD(), httpOpts);
LLSD body;
body["voice_server_type"] = "vivox";
result = httpAdapter->postAndSuspend(httpRequest, url, body, httpOpts);
if (sShuttingDown)
{

35
indra/newview/llvoicewebrtc.cpp
View File

@ -24,6 +24,7 @@
* $/LicenseInfo$
*/
#include <algorithm>
#include <format>
#include "llvoicewebrtc.h"
#include "llsdutil.h"
@ -279,7 +280,7 @@ void LLWebRTCVoiceClient::cleanUp()
LL_DEBUGS("Voice") << "Exiting" << LL_ENDL;
}
//---------------------------------------------------
// --------------------------------------------------
const LLVoiceVersionInfo& LLWebRTCVoiceClient::getVersion()
{
@ -299,6 +300,13 @@ void LLWebRTCVoiceClient::updateSettings()
setRenderDevice(outputDevice);
F32 mic_level = gSavedSettings.getF32("AudioLevelMic");
setMicGain(mic_level);
llwebrtc::LLWebRTCDeviceInterface::AudioConfig config;
config.mEchoCancellation = gSavedSettings.getBOOL("VoiceEchoCancellation");
config.mAGC = gSavedSettings.getBOOL("VoiceAutomaticGainControl");
config.mNoiseSuppressionLevel = (llwebrtc::LLWebRTCDeviceInterface::AudioConfig::ENoiseSuppressionLevel)gSavedSettings.getU32("VoiceNoiseSuppressionLevel");
mWebRTCDeviceInterface->setAudioConfig(config);
}
// Observers
@ -2480,6 +2488,28 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess(const LLSD &result
mWebRTCPeerConnectionInterface->AnswerAvailable(mRemoteChannelSDP);
}
static llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions getConnectionOptions()
{
llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions options;
llwebrtc::LLWebRTCPeerConnectionInterface::InitOptions::IceServers servers;
// TODO: Pull these from login
std::string grid = LLGridManager::getInstance()->getGridLoginID();
std::transform(grid.begin(), grid.end(), grid.begin(), [](unsigned char c){ return std::tolower(c); });
int num_servers = 2;
if (grid == "agni")
{
num_servers = 3;
}
for (int i=1; i <= num_servers; i++)
{
servers.mUrls.push_back(llformat("stun:stun%d.%s.secondlife.io:3478", i, grid.c_str()));
}
options.mServers.push_back(servers);
return options;
}
// Primary state machine for negotiating a single voice connection to the
// Secondlife WebRTC server.
bool LLVoiceWebRTCConnection::connectionStateMachine()
@ -2498,10 +2528,11 @@ bool LLVoiceWebRTCConnection::connectionStateMachine()
mTrickling = false;
mIceCompleted = false;
setVoiceConnectionState(VOICE_STATE_WAIT_FOR_SESSION_START);
// tell the webrtc library that we want a connection. The library will
// respond with an offer on a separate thread, which will cause
// the session state to change.
if (!mWebRTCPeerConnectionInterface->initializeConnection())
if (!mWebRTCPeerConnectionInterface->initializeConnection(getConnectionOptions()))
{
setVoiceConnectionState(VOICE_STATE_SESSION_RETRY);
}

2
indra/newview/llvoicewebrtc.h
View File

@ -272,7 +272,7 @@ public:
participantStatePtr_t findParticipantByID(const LLUUID& id);
static ptr_t matchSessionByChannelID(const std::string& channel_id);
void shutdownAllConnections();
void revive();

69
indra/newview/skins/default/xui/en/panel_preferences_sound.xml
View File

@ -340,7 +340,7 @@
follows="left|top"
top_delta="-6"
layout="topleft"
left_pad="20"
left_pad="10"
width="360"
height="40"
name="media_ear_location">
@ -422,7 +422,7 @@
control_name="VoiceEarLocation"
follows="left|top"
layout="topleft"
left_pad="20"
left_pad="10"
top_delta="-6"
width="360"
height="40"
@ -454,18 +454,38 @@
name="enable_lip_sync"
top_pad="10"
width="237"/>
<check_box
control_name="VoiceEchoCancellation"
height="15"
tool_tip="Check to enable voice echo cancellation"
label="Echo Cancellation"
layout="topleft"
left="260"
name="enable_echo_cancellation"
top_pad="-15"
width="200"/>
<check_box
follows="top|left"
enabled_control="EnableVoiceChat"
control_name="PushToTalkToggle"
height="15"
label="Toggle speak on/off when I press button in toolbar"
label="Toggle speak on/off with toolbar button"
layout="topleft"
left="20"
name="push_to_talk_toggle_check"
width="237"
tool_tip="When in toggle mode, press and release the trigger key ONCE to switch your microphone on or off. When not in toggle mode, the microphone broadcasts your voice only while the trigger is being held down."
top_pad="5"/>
<check_box
control_name="VoiceAutomaticGainControl"
height="15"
tool_tip="Check to enable automatic gain control"
label="Automatic Gain Control"
layout="topleft"
name="voice_automatic_gain_control"
left="260"
top_pad="-15"
width="200"/>
<check_box
name="gesture_audio_play_btn"
control_name="EnableGestureSounds"
@ -477,6 +497,45 @@
label="Play sounds from gestures"
top_pad="5"
left="20"/>
<text
layout="topleft"
height="15"
left="260"
top_pad="-12"
width="100"
name="noise_suppression_label">
Noise Suppression
</text>
<combo_box
control_name="VoiceNoiseSuppressionLevel"
enabled_control="AudioStreamingMedia"
layout="topleft"
height="23"
left_pad="10"
top_pad="-18"
name="noise_suppression_combo"
width="80">
<item
label="Off"
name="noise_suppression_none"
value="0"/>
<item
label="Low"
name="noise_suppression_low"
value="1"/>
<item
label="Moderate"
name="noise_suppression_moderate"
value="2"/>
<item
label="High"
name="noise_suppression_high"
value="3"/>
<item
label="Max"
name="noise_suppression_max"
value="4"/>
</combo_box>
<button
control_name="ShowDeviceSettings"
follows="left|top"
@ -485,9 +544,9 @@
label="Voice Input/Output devices"
layout="topleft"
left="20"
top_pad="9"
top_pad="0"
name="device_settings_btn"
width="230">
width="200">
</button>
<panel
layout="topleft"