mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-11-28 18:40:29 +00:00
b4fbd30b70
This change was a long time in the making ever since we obtained sample rate awareness in the system. Now, each client has its own sample rate, accessible via new IPC APIs, and the device sample rate is only accessible via the management interface. AudioServer takes care of resampling client streams into the device sample rate. Therefore, the main improvement introduced with this commit is full responsiveness to sample rate changes; all open audio programs will continue to play at correct speed with the audio resampled to the new device rate. The immediate benefits are manifold: - Gets rid of the legacy hardware sample rate IPC message in the non-managing client - Removes duplicate resampling and sample index rescaling code everywhere - Avoids potential sample index scaling bugs in SoundPlayer (which have happened many times before) and fixes a sample index scaling bug in aplay - Removes several FIXMEs - Reduces amount of sample copying in all applications (especially Piano, where this is critical), improving performance - Reduces number of resampling users, making future API changes (which will need to happen for correct resampling to be implemented) easier I also threw in a simple race condition fix for Piano's audio player loop.
343 lines
11 KiB
C++
343 lines
11 KiB
C++
/*
|
|
* Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include "AudioCodecPluginLadybird.h"
|
|
#include <AK/Endian.h>
|
|
#include <AK/MemoryStream.h>
|
|
#include <LibAudio/Loader.h>
|
|
#include <LibAudio/Sample.h>
|
|
#include <LibCore/SharedCircularQueue.h>
|
|
#include <QAudioFormat>
|
|
#include <QAudioSink>
|
|
#include <QByteArray>
|
|
#include <QMediaDevices>
|
|
#include <QThread>
|
|
|
|
namespace Ladybird {
|
|
|
|
static constexpr u32 UPDATE_RATE_MS = 10;
|
|
|
|
struct AudioTask {
|
|
enum class Type {
|
|
Stop,
|
|
Play,
|
|
Pause,
|
|
Seek,
|
|
Volume,
|
|
RecreateAudioDevice,
|
|
};
|
|
|
|
Type type;
|
|
Optional<double> data {};
|
|
};
|
|
|
|
using AudioTaskQueue = Core::SharedSingleProducerCircularQueue<AudioTask>;
|
|
|
|
class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work.
|
|
Q_OBJECT
|
|
|
|
public:
|
|
static ErrorOr<NonnullOwnPtr<AudioThread>> create(NonnullRefPtr<Audio::Loader> loader)
|
|
{
|
|
auto task_queue = TRY(AudioTaskQueue::create());
|
|
return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue)));
|
|
}
|
|
|
|
ErrorOr<void> stop()
|
|
{
|
|
TRY(queue_task({ AudioTask::Type::Stop }));
|
|
wait();
|
|
|
|
return {};
|
|
}
|
|
|
|
Duration duration() const
|
|
{
|
|
return m_duration;
|
|
}
|
|
|
|
ErrorOr<void> queue_task(AudioTask task)
|
|
{
|
|
return m_task_queue.blocking_enqueue(move(task), []() {
|
|
usleep(UPDATE_RATE_MS * 1000);
|
|
});
|
|
}
|
|
|
|
Q_SIGNALS:
|
|
void playback_position_updated(Duration);
|
|
|
|
private:
|
|
AudioThread(NonnullRefPtr<Audio::Loader> loader, AudioTaskQueue task_queue)
|
|
: m_loader(move(loader))
|
|
, m_task_queue(move(task_queue))
|
|
{
|
|
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
|
|
m_duration = Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
|
|
}
|
|
|
|
enum class Paused {
|
|
Yes,
|
|
No,
|
|
};
|
|
|
|
struct AudioDevice {
|
|
static AudioDevice create(Audio::Loader const& loader)
|
|
{
|
|
auto const& device_info = QMediaDevices::defaultAudioOutput();
|
|
|
|
auto format = device_info.preferredFormat();
|
|
format.setSampleRate(static_cast<int>(loader.sample_rate()));
|
|
format.setChannelCount(2);
|
|
|
|
auto audio_output = make<QAudioSink>(device_info, format);
|
|
return AudioDevice { move(audio_output) };
|
|
}
|
|
|
|
AudioDevice(AudioDevice&&) = default;
|
|
|
|
AudioDevice& operator=(AudioDevice&& device)
|
|
{
|
|
if (audio_output) {
|
|
audio_output->stop();
|
|
io_device = nullptr;
|
|
}
|
|
|
|
swap(audio_output, device.audio_output);
|
|
swap(io_device, device.io_device);
|
|
return *this;
|
|
}
|
|
|
|
~AudioDevice()
|
|
{
|
|
if (audio_output)
|
|
audio_output->stop();
|
|
}
|
|
|
|
OwnPtr<QAudioSink> audio_output;
|
|
QIODevice* io_device { nullptr };
|
|
|
|
private:
|
|
explicit AudioDevice(NonnullOwnPtr<QAudioSink> output)
|
|
: audio_output(move(output))
|
|
{
|
|
io_device = audio_output->start();
|
|
}
|
|
};
|
|
|
|
void run() override
|
|
{
|
|
auto devices = make<QMediaDevices>();
|
|
auto audio_device = AudioDevice::create(m_loader);
|
|
|
|
connect(devices, &QMediaDevices::audioOutputsChanged, this, [this]() {
|
|
queue_task({ AudioTask::Type::RecreateAudioDevice }).release_value_but_fixme_should_propagate_errors();
|
|
});
|
|
|
|
auto paused = Paused::Yes;
|
|
|
|
while (true) {
|
|
auto& audio_output = audio_device.audio_output;
|
|
auto* io_device = audio_device.io_device;
|
|
|
|
if (auto result = m_task_queue.dequeue(); result.is_error()) {
|
|
VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty);
|
|
} else {
|
|
auto task = result.release_value();
|
|
|
|
switch (task.type) {
|
|
case AudioTask::Type::Stop:
|
|
return;
|
|
|
|
case AudioTask::Type::Play:
|
|
audio_output->resume();
|
|
paused = Paused::No;
|
|
break;
|
|
|
|
case AudioTask::Type::Pause:
|
|
audio_output->suspend();
|
|
paused = Paused::Yes;
|
|
break;
|
|
|
|
case AudioTask::Type::Seek:
|
|
VERIFY(task.data.has_value());
|
|
m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration);
|
|
|
|
if (paused == Paused::Yes)
|
|
Q_EMIT playback_position_updated(m_position);
|
|
|
|
break;
|
|
|
|
case AudioTask::Type::Volume:
|
|
VERIFY(task.data.has_value());
|
|
audio_output->setVolume(*task.data);
|
|
break;
|
|
|
|
case AudioTask::Type::RecreateAudioDevice:
|
|
audio_device = AudioDevice::create(m_loader);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (paused == Paused::No) {
|
|
if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) {
|
|
// FIXME: Propagate the error to the HTMLMediaElement.
|
|
} else {
|
|
Q_EMIT playback_position_updated(m_position);
|
|
paused = result.value();
|
|
}
|
|
}
|
|
|
|
usleep(UPDATE_RATE_MS * 1000);
|
|
}
|
|
}
|
|
|
|
ErrorOr<Paused> play_next_samples(QAudioSink& audio_output, QIODevice& io_device)
|
|
{
|
|
bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
|
|
|
|
if (all_samples_loaded) {
|
|
audio_output.suspend();
|
|
(void)m_loader->reset();
|
|
|
|
m_position = m_duration;
|
|
return Paused::Yes;
|
|
}
|
|
|
|
auto bytes_available = audio_output.bytesFree();
|
|
auto bytes_per_sample = audio_output.format().bytesPerSample();
|
|
auto channel_count = audio_output.format().channelCount();
|
|
auto samples_to_load = bytes_available / bytes_per_sample / channel_count;
|
|
|
|
auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load));
|
|
enqueue_samples(audio_output, io_device, move(samples));
|
|
|
|
m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader);
|
|
return Paused::No;
|
|
}
|
|
|
|
void enqueue_samples(QAudioSink const& audio_output, QIODevice& io_device, FixedArray<Audio::Sample> samples)
|
|
{
|
|
auto buffer_size = samples.size() * audio_output.format().bytesPerSample() * audio_output.format().channelCount();
|
|
|
|
if (buffer_size > static_cast<size_t>(m_sample_buffer.size()))
|
|
m_sample_buffer.resize(buffer_size);
|
|
|
|
FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } };
|
|
|
|
for (auto const& sample : samples) {
|
|
switch (audio_output.format().sampleFormat()) {
|
|
case QAudioFormat::UInt8:
|
|
write_sample<u8>(stream, sample.left);
|
|
write_sample<u8>(stream, sample.right);
|
|
break;
|
|
case QAudioFormat::Int16:
|
|
write_sample<i16>(stream, sample.left);
|
|
write_sample<i16>(stream, sample.right);
|
|
break;
|
|
case QAudioFormat::Int32:
|
|
write_sample<i32>(stream, sample.left);
|
|
write_sample<i32>(stream, sample.right);
|
|
break;
|
|
case QAudioFormat::Float:
|
|
write_sample<float>(stream, sample.left);
|
|
write_sample<float>(stream, sample.right);
|
|
break;
|
|
default:
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
io_device.write(m_sample_buffer.data(), buffer_size);
|
|
}
|
|
|
|
template<typename T>
|
|
void write_sample(FixedMemoryStream& stream, float sample)
|
|
{
|
|
// The values that need to be written to the stream vary depending on the output channel format, and isn't
|
|
// particularly well documented. The value derivations performed below were adapted from a Qt example:
|
|
// https://code.qt.io/cgit/qt/qtmultimedia.git/tree/examples/multimedia/audiooutput/audiooutput.cpp?h=6.4.2#n46
|
|
LittleEndian<T> pcm;
|
|
|
|
if constexpr (IsSame<T, u8>)
|
|
pcm = static_cast<u8>((sample + 1.0f) / 2 * NumericLimits<u8>::max());
|
|
else if constexpr (IsSame<T, i16>)
|
|
pcm = static_cast<i16>(sample * NumericLimits<i16>::max());
|
|
else if constexpr (IsSame<T, i32>)
|
|
pcm = static_cast<i32>(sample * NumericLimits<i32>::max());
|
|
else if constexpr (IsSame<T, float>)
|
|
pcm = sample;
|
|
else
|
|
static_assert(DependentFalse<T>);
|
|
|
|
MUST(stream.write_value(pcm));
|
|
}
|
|
|
|
NonnullRefPtr<Audio::Loader> m_loader;
|
|
AudioTaskQueue m_task_queue;
|
|
|
|
QByteArray m_sample_buffer;
|
|
|
|
Duration m_duration;
|
|
Duration m_position;
|
|
};
|
|
|
|
ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> AudioCodecPluginLadybird::create(NonnullRefPtr<Audio::Loader> loader)
|
|
{
|
|
auto audio_thread = TRY(AudioThread::create(move(loader)));
|
|
audio_thread->start();
|
|
|
|
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(audio_thread)));
|
|
}
|
|
|
|
AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr<AudioThread> audio_thread)
|
|
: m_audio_thread(move(audio_thread))
|
|
{
|
|
connect(m_audio_thread, &AudioThread::playback_position_updated, this, [this](auto position) {
|
|
if (on_playback_position_updated)
|
|
on_playback_position_updated(position);
|
|
});
|
|
}
|
|
|
|
AudioCodecPluginLadybird::~AudioCodecPluginLadybird()
|
|
{
|
|
m_audio_thread->stop().release_value_but_fixme_should_propagate_errors();
|
|
}
|
|
|
|
void AudioCodecPluginLadybird::resume_playback()
|
|
{
|
|
m_audio_thread->queue_task({ AudioTask::Type::Play }).release_value_but_fixme_should_propagate_errors();
|
|
}
|
|
|
|
void AudioCodecPluginLadybird::pause_playback()
|
|
{
|
|
m_audio_thread->queue_task({ AudioTask::Type::Pause }).release_value_but_fixme_should_propagate_errors();
|
|
}
|
|
|
|
void AudioCodecPluginLadybird::set_volume(double volume)
|
|
{
|
|
|
|
AudioTask task { AudioTask::Type::Volume };
|
|
task.data = volume;
|
|
|
|
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
|
|
}
|
|
|
|
void AudioCodecPluginLadybird::seek(double position)
|
|
{
|
|
AudioTask task { AudioTask::Type::Seek };
|
|
task.data = position;
|
|
|
|
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
|
|
}
|
|
|
|
Duration AudioCodecPluginLadybird::duration()
|
|
{
|
|
return m_audio_thread->duration();
|
|
}
|
|
|
|
}
|
|
|
|
#include "AudioCodecPluginLadybird.moc"
|