livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use a custom audio processor to pull data for level determinations, which will happen after AGC

master
Roxie Linden 2024-02-05 14:42:21 -08:00
parent 1e882869dc
commit 65b5a6e12d
2 changed files with 69 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
indra/llwebrtc/llwebrtc.cpp
View File

@ -27,6 +27,7 @@
#include "llwebrtc_impl.h"
#include <algorithm>
#include <format>
#include <string.h>
#include "api/audio_codecs/audio_decoder_factory.h"
#include "api/audio_codecs/audio_encoder_factory.h"
@ -34,30 +35,54 @@
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "api/media_stream_interface.h"
#include "api/media_stream_track.h"
#include "modules/audio_processing/audio_buffer.h"
namespace llwebrtc
{
const float VOLUME_SCALE_WEBRTC = 100;
LLCustomProcessor::LLCustomProcessor() :
mSampleRateHz(0),
mNumChannels(0)
{
memset(mSumVector, sizeof(mSumVector), 0);
}
LLAudioDeviceObserver::LLAudioDeviceObserver() : mMicrophoneEnergy(0.0), mSumVector {0} {}
void LLCustomProcessor::Initialize(int sample_rate_hz, int num_channels)
{
mSampleRateHz = sample_rate_hz;
mNumChannels = num_channels;
memset(mSumVector, sizeof(mSumVector), 0);
}
float LLAudioDeviceObserver::getMicrophoneEnergy() { return mMicrophoneEnergy; }
void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec)
void LLCustomProcessor::Process(webrtc::AudioBuffer *audio_in)
{
float energy = 0;
const short *samples = (const short *) audio_samples;
for (size_t index = 0; index < num_samples * num_channels; index++)
webrtc::StreamConfig stream_config;
stream_config.set_sample_rate_hz(mSampleRateHz);
stream_config.set_num_channels(mNumChannels);
std::vector<float *> frame;
std::vector<float> frame_samples;
if (audio_in->num_channels() < 1 || audio_in->num_frames() < 480)
{
float sample = (static_cast<float>(samples[index]) / (float) 32767);
return;
}
frame_samples.resize(stream_config.num_samples());
frame.resize(stream_config.num_channels());
for (size_t ch = 0; ch < stream_config.num_channels(); ++ch)
{
frame[ch] = &(frame_samples)[ch * stream_config.num_frames()];
}
audio_in->CopyTo(stream_config, &frame[0]);
float energy = 0;
for (size_t index = 0; index < stream_config.num_samples(); index++)
{
float sample = frame_samples[index];
energy += sample * sample;
}
// smooth it.
size_t buffer_size = sizeof(mSumVector) / sizeof(mSumVector[0]);
float totalSum = 0;
@ -69,15 +94,7 @@ void LLAudioDeviceObserver::OnCaptureData(const void *audio_samples,
}
mSumVector[i] = energy;
totalSum += energy;
mMicrophoneEnergy = std::sqrt(totalSum / (num_samples * buffer_size));
}
void LLAudioDeviceObserver::OnRenderData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec)
{
mMicrophoneEnergy = std::sqrt(totalSum / (stream_config.num_samples() * buffer_size));
}
void LLWebRTCImpl::init()
@ -104,11 +121,9 @@ void LLWebRTCImpl::init()
mWorkerThread->PostTask(
[this]()
{
mTuningAudioDeviceObserver = new LLAudioDeviceObserver;
mTuningDeviceModule =
webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(),
std::unique_ptr<webrtc::AudioDeviceDataObserver>(mTuningAudioDeviceObserver));
mTuningDeviceModule = webrtc::CreateAudioDeviceWithDataObserver(
webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(), nullptr);
mTuningDeviceModule->Init();
mTuningDeviceModule->SetStereoRecording(true);
mTuningDeviceModule->SetStereoPlayout(true);
@ -134,11 +149,9 @@ void LLWebRTCImpl::init()
mWorkerThread->BlockingCall(
[this]()
{
mPeerAudioDeviceObserver = new LLAudioDeviceObserver;
mPeerDeviceModule =
webrtc::CreateAudioDeviceWithDataObserver(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio,
mTaskQueueFactory.get(),
std::unique_ptr<webrtc::AudioDeviceDataObserver>(mPeerAudioDeviceObserver));
mTaskQueueFactory.get(), nullptr);
mPeerDeviceModule->Init();
mPeerDeviceModule->SetPlayoutDevice(mPlayoutDevice);
mPeerDeviceModule->SetRecordingDevice(mRecordingDevice);
@ -151,7 +164,11 @@ void LLWebRTCImpl::init()
mPeerDeviceModule->InitPlayout();
});
rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create();
mCustomProcessor = new LLCustomProcessor;
webrtc::AudioProcessingBuilder apb;
apb.SetCapturePostProcessing(std::unique_ptr<webrtc::CustomProcessing>(mCustomProcessor));
rtc::scoped_refptr<webrtc::AudioProcessing> apm = apb.Create();
webrtc::AudioProcessing::Config apm_config;
apm_config.echo_canceller.enabled = true;
apm_config.echo_canceller.mobile_mode = false;
@ -454,9 +471,9 @@ void LLWebRTCImpl::setTuningMode(bool enable)
}
}
float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mTuningAudioDeviceObserver->getMicrophoneEnergy()); }
float LLWebRTCImpl::getTuningAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); }
float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mPeerAudioDeviceObserver->getMicrophoneEnergy()); }
float LLWebRTCImpl::getPeerAudioLevel() { return -20 * log10f(mCustomProcessor->getMicrophoneEnergy()); }
//
// Helpers

37
indra/llwebrtc/llwebrtc_impl.h
View File

@ -65,27 +65,27 @@ namespace llwebrtc
class LLWebRTCPeerConnectionImpl;
class LLAudioDeviceObserver : public webrtc::AudioDeviceDataObserver
class LLCustomProcessor : public webrtc::CustomProcessing
{
public:
LLAudioDeviceObserver();
LLCustomProcessor();
~LLCustomProcessor() override {}
float getMicrophoneEnergy();
// (Re-) Initializes the submodule.
void Initialize(int sample_rate_hz, int num_channels) override;
// Analyzes the given capture or render signal.
void Process(webrtc::AudioBuffer *audio) override;
// Returns a string representation of the module state.
std::string ToString() const override { return ""; }
void OnCaptureData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec) override;
void OnRenderData(const void *audio_samples,
const size_t num_samples,
const size_t bytes_per_sample,
const size_t num_channels,
const uint32_t samples_per_sec) override;
float getMicrophoneEnergy() { return mMicrophoneEnergy; }
protected:
float mSumVector[30]; // 300 ms of smoothing
static const int NUM_PACKETS_TO_FILTER = 30; // 300 ms of smoothing
int mSampleRateHz;
int mNumChannels;
float mSumVector[NUM_PACKETS_TO_FILTER];
float mMicrophoneEnergy;
};
@ -93,7 +93,7 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
{
public:
LLWebRTCImpl() :
mTuningAudioDeviceObserver(nullptr), mPeerAudioDeviceObserver(nullptr), mMute(true)
mCustomProcessor(nullptr), mMute(true)
{
}
~LLWebRTCImpl() {}
@ -189,9 +189,8 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceS
int32_t mPlayoutDevice;
int32_t mRecordingDevice;
bool mMute;
LLAudioDeviceObserver * mTuningAudioDeviceObserver;
LLAudioDeviceObserver * mPeerAudioDeviceObserver;
LLCustomProcessor * mCustomProcessor;
// peer connections
std::vector<rtc::scoped_refptr<LLWebRTCPeerConnectionImpl>> mPeerConnections;