ladybird/Ladybird/AudioCodecPluginLadybird.cpp
kleines Filmröllchen b4fbd30b70 AudioServer+Userland: Decouple client sample rates from device rate
This change was a long time in the making ever since we obtained sample
rate awareness in the system. Now, each client has its own sample rate,
accessible via new IPC APIs, and the device sample rate is only
accessible via the management interface. AudioServer takes care of
resampling client streams into the device sample rate. Therefore, the
main improvement introduced with this commit is full responsiveness to
sample rate changes; all open audio programs will continue to play at
correct speed with the audio resampled to the new device rate.

The immediate benefits are manifold:
- Gets rid of the legacy hardware sample rate IPC message in the
  non-managing client
- Removes duplicate resampling and sample index rescaling code
  everywhere
- Avoids potential sample index scaling bugs in SoundPlayer (which have
  happened many times before) and fixes a sample index scaling bug in
  aplay
- Removes several FIXMEs
- Reduces amount of sample copying in all applications (especially
  Piano, where this is critical), improving performance
- Reduces number of resampling users, making future API changes (which
  will need to happen for correct resampling to be implemented) easier

I also threw in a simple race condition fix for Piano's audio player
loop.
2023-07-01 23:27:24 +01:00

343 lines
11 KiB
C++

/*
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "AudioCodecPluginLadybird.h"
#include <AK/Endian.h>
#include <AK/MemoryStream.h>
#include <LibAudio/Loader.h>
#include <LibAudio/Sample.h>
#include <LibCore/SharedCircularQueue.h>
#include <QAudioFormat>
#include <QAudioSink>
#include <QByteArray>
#include <QMediaDevices>
#include <QThread>
namespace Ladybird {
static constexpr u32 UPDATE_RATE_MS = 10;
struct AudioTask {
enum class Type {
Stop,
Play,
Pause,
Seek,
Volume,
RecreateAudioDevice,
};
Type type;
Optional<double> data {};
};
using AudioTaskQueue = Core::SharedSingleProducerCircularQueue<AudioTask>;
class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work.
Q_OBJECT
public:
static ErrorOr<NonnullOwnPtr<AudioThread>> create(NonnullRefPtr<Audio::Loader> loader)
{
auto task_queue = TRY(AudioTaskQueue::create());
return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue)));
}
ErrorOr<void> stop()
{
TRY(queue_task({ AudioTask::Type::Stop }));
wait();
return {};
}
Duration duration() const
{
return m_duration;
}
ErrorOr<void> queue_task(AudioTask task)
{
return m_task_queue.blocking_enqueue(move(task), []() {
usleep(UPDATE_RATE_MS * 1000);
});
}
Q_SIGNALS:
void playback_position_updated(Duration);
private:
AudioThread(NonnullRefPtr<Audio::Loader> loader, AudioTaskQueue task_queue)
: m_loader(move(loader))
, m_task_queue(move(task_queue))
{
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
m_duration = Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
}
enum class Paused {
Yes,
No,
};
struct AudioDevice {
static AudioDevice create(Audio::Loader const& loader)
{
auto const& device_info = QMediaDevices::defaultAudioOutput();
auto format = device_info.preferredFormat();
format.setSampleRate(static_cast<int>(loader.sample_rate()));
format.setChannelCount(2);
auto audio_output = make<QAudioSink>(device_info, format);
return AudioDevice { move(audio_output) };
}
AudioDevice(AudioDevice&&) = default;
AudioDevice& operator=(AudioDevice&& device)
{
if (audio_output) {
audio_output->stop();
io_device = nullptr;
}
swap(audio_output, device.audio_output);
swap(io_device, device.io_device);
return *this;
}
~AudioDevice()
{
if (audio_output)
audio_output->stop();
}
OwnPtr<QAudioSink> audio_output;
QIODevice* io_device { nullptr };
private:
explicit AudioDevice(NonnullOwnPtr<QAudioSink> output)
: audio_output(move(output))
{
io_device = audio_output->start();
}
};
void run() override
{
auto devices = make<QMediaDevices>();
auto audio_device = AudioDevice::create(m_loader);
connect(devices, &QMediaDevices::audioOutputsChanged, this, [this]() {
queue_task({ AudioTask::Type::RecreateAudioDevice }).release_value_but_fixme_should_propagate_errors();
});
auto paused = Paused::Yes;
while (true) {
auto& audio_output = audio_device.audio_output;
auto* io_device = audio_device.io_device;
if (auto result = m_task_queue.dequeue(); result.is_error()) {
VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty);
} else {
auto task = result.release_value();
switch (task.type) {
case AudioTask::Type::Stop:
return;
case AudioTask::Type::Play:
audio_output->resume();
paused = Paused::No;
break;
case AudioTask::Type::Pause:
audio_output->suspend();
paused = Paused::Yes;
break;
case AudioTask::Type::Seek:
VERIFY(task.data.has_value());
m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration);
if (paused == Paused::Yes)
Q_EMIT playback_position_updated(m_position);
break;
case AudioTask::Type::Volume:
VERIFY(task.data.has_value());
audio_output->setVolume(*task.data);
break;
case AudioTask::Type::RecreateAudioDevice:
audio_device = AudioDevice::create(m_loader);
continue;
}
}
if (paused == Paused::No) {
if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) {
// FIXME: Propagate the error to the HTMLMediaElement.
} else {
Q_EMIT playback_position_updated(m_position);
paused = result.value();
}
}
usleep(UPDATE_RATE_MS * 1000);
}
}
ErrorOr<Paused> play_next_samples(QAudioSink& audio_output, QIODevice& io_device)
{
bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
if (all_samples_loaded) {
audio_output.suspend();
(void)m_loader->reset();
m_position = m_duration;
return Paused::Yes;
}
auto bytes_available = audio_output.bytesFree();
auto bytes_per_sample = audio_output.format().bytesPerSample();
auto channel_count = audio_output.format().channelCount();
auto samples_to_load = bytes_available / bytes_per_sample / channel_count;
auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load));
enqueue_samples(audio_output, io_device, move(samples));
m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader);
return Paused::No;
}
void enqueue_samples(QAudioSink const& audio_output, QIODevice& io_device, FixedArray<Audio::Sample> samples)
{
auto buffer_size = samples.size() * audio_output.format().bytesPerSample() * audio_output.format().channelCount();
if (buffer_size > static_cast<size_t>(m_sample_buffer.size()))
m_sample_buffer.resize(buffer_size);
FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } };
for (auto const& sample : samples) {
switch (audio_output.format().sampleFormat()) {
case QAudioFormat::UInt8:
write_sample<u8>(stream, sample.left);
write_sample<u8>(stream, sample.right);
break;
case QAudioFormat::Int16:
write_sample<i16>(stream, sample.left);
write_sample<i16>(stream, sample.right);
break;
case QAudioFormat::Int32:
write_sample<i32>(stream, sample.left);
write_sample<i32>(stream, sample.right);
break;
case QAudioFormat::Float:
write_sample<float>(stream, sample.left);
write_sample<float>(stream, sample.right);
break;
default:
VERIFY_NOT_REACHED();
}
}
io_device.write(m_sample_buffer.data(), buffer_size);
}
template<typename T>
void write_sample(FixedMemoryStream& stream, float sample)
{
// The values that need to be written to the stream vary depending on the output channel format, and isn't
// particularly well documented. The value derivations performed below were adapted from a Qt example:
// https://code.qt.io/cgit/qt/qtmultimedia.git/tree/examples/multimedia/audiooutput/audiooutput.cpp?h=6.4.2#n46
LittleEndian<T> pcm;
if constexpr (IsSame<T, u8>)
pcm = static_cast<u8>((sample + 1.0f) / 2 * NumericLimits<u8>::max());
else if constexpr (IsSame<T, i16>)
pcm = static_cast<i16>(sample * NumericLimits<i16>::max());
else if constexpr (IsSame<T, i32>)
pcm = static_cast<i32>(sample * NumericLimits<i32>::max());
else if constexpr (IsSame<T, float>)
pcm = sample;
else
static_assert(DependentFalse<T>);
MUST(stream.write_value(pcm));
}
NonnullRefPtr<Audio::Loader> m_loader;
AudioTaskQueue m_task_queue;
QByteArray m_sample_buffer;
Duration m_duration;
Duration m_position;
};
ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> AudioCodecPluginLadybird::create(NonnullRefPtr<Audio::Loader> loader)
{
auto audio_thread = TRY(AudioThread::create(move(loader)));
audio_thread->start();
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(audio_thread)));
}
AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr<AudioThread> audio_thread)
: m_audio_thread(move(audio_thread))
{
connect(m_audio_thread, &AudioThread::playback_position_updated, this, [this](auto position) {
if (on_playback_position_updated)
on_playback_position_updated(position);
});
}
AudioCodecPluginLadybird::~AudioCodecPluginLadybird()
{
m_audio_thread->stop().release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::resume_playback()
{
m_audio_thread->queue_task({ AudioTask::Type::Play }).release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::pause_playback()
{
m_audio_thread->queue_task({ AudioTask::Type::Pause }).release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::set_volume(double volume)
{
AudioTask task { AudioTask::Type::Volume };
task.data = volume;
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::seek(double position)
{
AudioTask task { AudioTask::Type::Seek };
task.data = position;
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
}
Duration AudioCodecPluginLadybird::duration()
{
return m_audio_thread->duration();
}
}
#include "AudioCodecPluginLadybird.moc"