Ladybird+LibWeb+WebConent: Drive audio in Ladybird off the main thread

The main thread in the WebContent process is often busy with layout and
running JavaScript. This can cause audio to sound jittery and crack. To
avoid this behavior, we now drive audio on a secondary thread.

Note: Browser on Serenity uses AudioServer, the connection for which is
already handled on a secondary thread within LibAudio. So this only
applies to Lagom.

Rather than using LibThreading, our hands are tied to QThread for now.
Internally, the Qt media objects use a QTimer, which is forbidden from
running on a thread that is not a QThread (the debug console is spammed
with messages pointing this out). Ideally, in the future AudioServer
will be able to run for non-Serenity platforms, and most of this can be
aligned with the Serenity implementation.
This commit is contained in:
Timothy Flynn 2023-06-20 12:43:04 -04:00 committed by Andreas Kling
parent 0fd35b4dd8
commit 1c4dd0caad
Notes: sideshowbarker 2024-07-16 21:39:23 +09:00
10 changed files with 383 additions and 205 deletions

View file

@ -7,88 +7,258 @@
#include "AudioCodecPluginLadybird.h" #include "AudioCodecPluginLadybird.h"
#include <AK/Endian.h> #include <AK/Endian.h>
#include <AK/MemoryStream.h> #include <AK/MemoryStream.h>
#include <LibAudio/Loader.h>
#include <LibAudio/Sample.h> #include <LibAudio/Sample.h>
#include <LibCore/SharedCircularQueue.h>
#include <QAudioFormat> #include <QAudioFormat>
#include <QAudioSink> #include <QAudioSink>
#include <QBuffer> #include <QByteArray>
#include <QMediaDevices> #include <QMediaDevices>
#include <QThread>
namespace Ladybird { namespace Ladybird {
ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> AudioCodecPluginLadybird::create() static constexpr u32 UPDATE_RATE_MS = 50;
{
auto devices = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QMediaDevices()));
auto const& device_info = devices->defaultAudioOutput();
auto format = device_info.preferredFormat(); struct AudioTask {
format.setSampleFormat(QAudioFormat::Int16); enum class Type {
format.setChannelCount(2); Stop,
Play,
Pause,
Seek,
Volume,
};
if (!device_info.isFormatSupported(format)) Type type;
return Error::from_string_literal("Audio device format not supported"); Optional<double> data {};
};
auto audio_output = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QAudioSink(device_info, format))); using AudioTaskQueue = Core::SharedSingleProducerCircularQueue<AudioTask>;
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(devices), move(audio_output))); class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work.
} Q_OBJECT
AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr<QMediaDevices> devices, NonnullOwnPtr<QAudioSink> audio_output) public:
: m_devices(move(devices)) static ErrorOr<NonnullOwnPtr<AudioThread>> create(NonnullRefPtr<Audio::Loader> loader)
, m_audio_output(move(audio_output)) {
, m_io_device(m_audio_output->start()) auto task_queue = TRY(AudioTaskQueue::create());
{ return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue)));
}
AudioCodecPluginLadybird::~AudioCodecPluginLadybird() = default;
size_t AudioCodecPluginLadybird::device_sample_rate()
{
return m_audio_output->format().sampleRate();
}
void AudioCodecPluginLadybird::enqueue_samples(FixedArray<Audio::Sample> samples)
{
QByteArray buffer;
buffer.resize(samples.size() * 2 * sizeof(u16));
FixedMemoryStream stream { Bytes { buffer.data(), static_cast<size_t>(buffer.size()) } };
for (auto& sample : samples) {
LittleEndian<i16> pcm;
pcm = static_cast<i16>(sample.left * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
pcm = static_cast<i16>(sample.right * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
} }
m_io_device->write(buffer.data(), buffer.size()); ErrorOr<void> stop()
{
TRY(queue_task({ AudioTask::Type::Stop }));
wait();
return {};
}
Duration duration() const
{
return m_duration;
}
ErrorOr<void> queue_task(AudioTask task)
{
return m_task_queue.blocking_enqueue(move(task), []() {
usleep(UPDATE_RATE_MS * 1000);
});
}
Q_SIGNALS:
void playback_position_updated(Duration);
private:
AudioThread(NonnullRefPtr<Audio::Loader> loader, AudioTaskQueue task_queue)
: m_loader(move(loader))
, m_task_queue(move(task_queue))
{
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
m_duration = Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
m_samples_to_load_per_buffer = static_cast<size_t>(UPDATE_RATE_MS / 1000.0 * static_cast<double>(m_loader->sample_rate()));
}
enum class Paused {
Yes,
No,
};
void run() override
{
auto devices = make<QMediaDevices>();
auto const& device_info = devices->defaultAudioOutput();
auto format = device_info.preferredFormat();
format.setSampleFormat(QAudioFormat::Int16);
format.setChannelCount(2);
auto audio_output = make<QAudioSink>(device_info, format);
auto* io_device = audio_output->start();
auto paused = Paused::Yes;
while (true) {
if (auto result = m_task_queue.dequeue(); result.is_error()) {
VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty);
} else {
auto task = result.release_value();
switch (task.type) {
case AudioTask::Type::Stop:
return;
case AudioTask::Type::Play:
audio_output->resume();
paused = Paused::No;
break;
case AudioTask::Type::Pause:
audio_output->suspend();
paused = Paused::Yes;
break;
case AudioTask::Type::Seek: {
VERIFY(task.data.has_value());
auto position = *task.data;
auto duration = static_cast<double>(this->duration().to_milliseconds()) / 1000.0;
position = position / duration * static_cast<double>(m_loader->total_samples());
m_loader->seek(static_cast<int>(position)).release_value_but_fixme_should_propagate_errors();
if (paused == Paused::Yes) {
m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader, audio_output->format().sampleRate());
Q_EMIT playback_position_updated(m_position);
}
break;
}
case AudioTask::Type::Volume:
VERIFY(task.data.has_value());
audio_output->setVolume(*task.data);
break;
}
}
if (paused == Paused::No) {
if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) {
// FIXME: Propagate the error to the HTMLMediaElement.
} else {
Q_EMIT playback_position_updated(m_position);
paused = result.value();
}
}
usleep(UPDATE_RATE_MS * 1000);
}
}
ErrorOr<Paused> play_next_samples(QAudioSink& audio_output, QIODevice& io_device)
{
bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
if (all_samples_loaded) {
audio_output.suspend();
(void)m_loader->reset();
m_position = m_duration;
return Paused::Yes;
}
auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, m_samples_to_load_per_buffer, audio_output.format().sampleRate()));
enqueue_samples(io_device, move(samples));
m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader, audio_output.format().sampleRate());
return Paused::No;
}
void enqueue_samples(QIODevice& io_device, FixedArray<Audio::Sample> samples)
{
auto buffer_size = samples.size() * 2 * sizeof(u16);
if (buffer_size > static_cast<size_t>(m_sample_buffer.size()))
m_sample_buffer.resize(buffer_size);
FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } };
for (auto& sample : samples) {
LittleEndian<i16> pcm;
pcm = static_cast<i16>(sample.left * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
pcm = static_cast<i16>(sample.right * NumericLimits<i16>::max());
MUST(stream.write_value(pcm));
}
io_device.write(m_sample_buffer.data(), buffer_size);
}
NonnullRefPtr<Audio::Loader> m_loader;
AudioTaskQueue m_task_queue;
size_t m_samples_to_load_per_buffer { 0 };
QByteArray m_sample_buffer;
Duration m_duration;
Duration m_position;
};
ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> AudioCodecPluginLadybird::create(NonnullRefPtr<Audio::Loader> loader)
{
auto audio_thread = TRY(AudioThread::create(move(loader)));
audio_thread->start();
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(audio_thread)));
} }
size_t AudioCodecPluginLadybird::remaining_samples() const AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr<AudioThread> audio_thread)
: m_audio_thread(move(audio_thread))
{ {
return 0; connect(m_audio_thread, &AudioThread::playback_position_updated, this, [this](auto position) {
if (on_playback_position_updated)
on_playback_position_updated(position);
});
}
AudioCodecPluginLadybird::~AudioCodecPluginLadybird()
{
m_audio_thread->stop().release_value_but_fixme_should_propagate_errors();
} }
void AudioCodecPluginLadybird::resume_playback() void AudioCodecPluginLadybird::resume_playback()
{ {
m_audio_output->resume(); m_audio_thread->queue_task({ AudioTask::Type::Play }).release_value_but_fixme_should_propagate_errors();
} }
void AudioCodecPluginLadybird::pause_playback() void AudioCodecPluginLadybird::pause_playback()
{ {
m_audio_output->suspend(); m_audio_thread->queue_task({ AudioTask::Type::Pause }).release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::playback_ended()
{
m_audio_output->suspend();
} }
void AudioCodecPluginLadybird::set_volume(double volume) void AudioCodecPluginLadybird::set_volume(double volume)
{ {
m_audio_output->setVolume(volume);
AudioTask task { AudioTask::Type::Volume };
task.data = volume;
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
}
void AudioCodecPluginLadybird::seek(double position)
{
AudioTask task { AudioTask::Type::Seek };
task.data = position;
m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors();
}
Duration AudioCodecPluginLadybird::duration()
{
return m_audio_thread->duration();
} }
} }
#include "AudioCodecPluginLadybird.moc"

View file

@ -8,37 +8,35 @@
#include <AK/Error.h> #include <AK/Error.h>
#include <AK/NonnullOwnPtr.h> #include <AK/NonnullOwnPtr.h>
#include <AK/NonnullRefPtr.h>
#include <LibAudio/Forward.h> #include <LibAudio/Forward.h>
#include <LibWeb/Platform/AudioCodecPlugin.h> #include <LibWeb/Platform/AudioCodecPlugin.h>
#include <QObject>
class QAudioSink;
class QIODevice;
class QMediaDevices;
namespace Ladybird { namespace Ladybird {
class AudioCodecPluginLadybird final : public Web::Platform::AudioCodecPlugin { class AudioThread;
class AudioCodecPluginLadybird final
: public QObject
, public Web::Platform::AudioCodecPlugin {
Q_OBJECT
public: public:
static ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> create(); static ErrorOr<NonnullOwnPtr<AudioCodecPluginLadybird>> create(NonnullRefPtr<Audio::Loader>);
virtual ~AudioCodecPluginLadybird() override; virtual ~AudioCodecPluginLadybird() override;
virtual size_t device_sample_rate() override;
virtual void enqueue_samples(FixedArray<Audio::Sample>) override;
virtual size_t remaining_samples() const override;
virtual void resume_playback() override; virtual void resume_playback() override;
virtual void pause_playback() override; virtual void pause_playback() override;
virtual void playback_ended() override;
virtual void set_volume(double) override; virtual void set_volume(double) override;
virtual void seek(double) override;
virtual Duration duration() override;
private: private:
AudioCodecPluginLadybird(NonnullOwnPtr<QMediaDevices>, NonnullOwnPtr<QAudioSink>); explicit AudioCodecPluginLadybird(NonnullOwnPtr<AudioThread>);
NonnullOwnPtr<QMediaDevices> m_devices; NonnullOwnPtr<AudioThread> m_audio_thread;
NonnullOwnPtr<QAudioSink> m_audio_output;
QIODevice* m_io_device { nullptr };
}; };
} }

View file

@ -13,6 +13,7 @@
#include "../WebSocketClientManagerLadybird.h" #include "../WebSocketClientManagerLadybird.h"
#include <AK/LexicalPath.h> #include <AK/LexicalPath.h>
#include <AK/Platform.h> #include <AK/Platform.h>
#include <LibAudio/Loader.h>
#include <LibCore/ArgsParser.h> #include <LibCore/ArgsParser.h>
#include <LibCore/EventLoop.h> #include <LibCore/EventLoop.h>
#include <LibCore/LocalServer.h> #include <LibCore/LocalServer.h>
@ -59,8 +60,8 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity); Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity);
Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird); Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird);
Web::Platform::AudioCodecPlugin::install_creation_hook([] { Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) {
return Ladybird::AudioCodecPluginLadybird::create(); return Ladybird::AudioCodecPluginLadybird::create(move(loader));
}); });
Web::ResourceLoader::initialize(RequestManagerQt::create()); Web::ResourceLoader::initialize(RequestManagerQt::create());

View file

@ -5,10 +5,7 @@
*/ */
#include <AK/IDAllocator.h> #include <AK/IDAllocator.h>
#include <AK/Time.h>
#include <LibAudio/Loader.h> #include <LibAudio/Loader.h>
#include <LibAudio/Resampler.h>
#include <LibAudio/Sample.h>
#include <LibJS/Runtime/Realm.h> #include <LibJS/Runtime/Realm.h>
#include <LibJS/Runtime/VM.h> #include <LibJS/Runtime/VM.h>
#include <LibWeb/Bindings/AudioTrackPrototype.h> #include <LibWeb/Bindings/AudioTrackPrototype.h>
@ -20,25 +17,23 @@
#include <LibWeb/HTML/HTMLMediaElement.h> #include <LibWeb/HTML/HTMLMediaElement.h>
#include <LibWeb/Layout/Node.h> #include <LibWeb/Layout/Node.h>
#include <LibWeb/Platform/AudioCodecPlugin.h> #include <LibWeb/Platform/AudioCodecPlugin.h>
#include <LibWeb/Platform/Timer.h>
namespace Web::HTML { namespace Web::HTML {
static IDAllocator s_audio_track_id_allocator; static IDAllocator s_audio_track_id_allocator;
// Number of milliseconds of audio data contained in each audio buffer
static constexpr u32 BUFFER_SIZE_MS = 50;
AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullRefPtr<Audio::Loader> loader) AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullRefPtr<Audio::Loader> loader)
: PlatformObject(realm) : PlatformObject(realm)
, m_media_element(media_element) , m_media_element(media_element)
, m_audio_plugin(Platform::AudioCodecPlugin::create().release_value_but_fixme_should_propagate_errors()) , m_audio_plugin(Platform::AudioCodecPlugin::create(move(loader)).release_value_but_fixme_should_propagate_errors())
, m_loader(move(loader))
, m_sample_timer(Platform::Timer::create_repeating(BUFFER_SIZE_MS, [this]() {
play_next_samples();
}))
{ {
m_audio_plugin->device_sample_rate(); m_audio_plugin->on_playback_position_updated = [this](auto position) {
if (auto* layout_node = m_media_element->layout_node())
layout_node->set_needs_display();
auto playback_position = static_cast<double>(position.to_milliseconds()) / 1000.0;
m_media_element->set_current_playback_position(playback_position);
};
} }
AudioTrack::~AudioTrack() AudioTrack::~AudioTrack()
@ -63,30 +58,16 @@ JS::ThrowCompletionOr<void> AudioTrack::initialize(JS::Realm& realm)
void AudioTrack::play(Badge<HTMLAudioElement>) void AudioTrack::play(Badge<HTMLAudioElement>)
{ {
m_audio_plugin->resume_playback(); m_audio_plugin->resume_playback();
m_sample_timer->start();
} }
void AudioTrack::pause(Badge<HTMLAudioElement>) void AudioTrack::pause(Badge<HTMLAudioElement>)
{ {
m_audio_plugin->pause_playback(); m_audio_plugin->pause_playback();
m_sample_timer->stop();
} }
Duration AudioTrack::position() const Duration AudioTrack::duration()
{ {
auto samples_played = static_cast<double>(m_loader->loaded_samples()); return m_audio_plugin->duration();
auto sample_rate = static_cast<double>(m_loader->sample_rate());
auto source_to_device_ratio = sample_rate / static_cast<double>(m_audio_plugin->device_sample_rate());
samples_played *= source_to_device_ratio;
return Duration::from_milliseconds(static_cast<i64>(samples_played / sample_rate * 1000.0));
}
Duration AudioTrack::duration() const
{
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
return Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
} }
void AudioTrack::seek(double position, MediaSeekMode seek_mode) void AudioTrack::seek(double position, MediaSeekMode seek_mode)
@ -94,11 +75,7 @@ void AudioTrack::seek(double position, MediaSeekMode seek_mode)
// FIXME: Implement seeking mode. // FIXME: Implement seeking mode.
(void)seek_mode; (void)seek_mode;
auto duration = static_cast<double>(this->duration().to_milliseconds()) / 1000.0; m_audio_plugin->seek(position);
position = position / duration * static_cast<double>(m_loader->total_samples());
m_loader->seek(position).release_value_but_fixme_should_propagate_errors();
m_media_element->set_current_playback_position(this->position().to_milliseconds() / 1000.0);
} }
void AudioTrack::update_volume() void AudioTrack::update_volume()
@ -134,48 +111,4 @@ void AudioTrack::set_enabled(bool enabled)
m_enabled = enabled; m_enabled = enabled;
} }
Optional<FixedArray<Audio::Sample>> AudioTrack::get_next_samples()
{
bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
bool audio_server_done = m_audio_plugin->remaining_samples() == 0;
if (all_samples_loaded && audio_server_done)
return {};
auto samples_to_load_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0f * static_cast<float>(m_loader->sample_rate()));
auto buffer_or_error = m_loader->get_more_samples(samples_to_load_per_buffer);
if (buffer_or_error.is_error()) {
dbgln("Error while loading samples: {}", buffer_or_error.error().description);
return {};
}
return buffer_or_error.release_value();
}
void AudioTrack::play_next_samples()
{
if (auto* layout_node = m_media_element->layout_node())
layout_node->set_needs_display();
auto samples = get_next_samples();
if (!samples.has_value()) {
m_audio_plugin->playback_ended();
(void)m_loader->reset();
auto playback_position = static_cast<double>(duration().to_milliseconds()) / 1000.0;
m_media_element->set_current_playback_position(playback_position);
return;
}
Audio::ResampleHelper<Audio::Sample> resampler(m_loader->sample_rate(), m_audio_plugin->device_sample_rate());
auto resampled = FixedArray<Audio::Sample>::create(resampler.resample(samples.release_value()).span()).release_value_but_fixme_should_propagate_errors();
m_audio_plugin->enqueue_samples(move(resampled));
auto playback_position = static_cast<double>(position().to_milliseconds()) / 1000.0;
m_media_element->set_current_playback_position(playback_position);
}
} }

View file

@ -24,8 +24,7 @@ public:
void play(Badge<HTMLAudioElement>); void play(Badge<HTMLAudioElement>);
void pause(Badge<HTMLAudioElement>); void pause(Badge<HTMLAudioElement>);
Duration position() const; Duration duration();
Duration duration() const;
void seek(double, MediaSeekMode); void seek(double, MediaSeekMode);
void update_volume(); void update_volume();
@ -44,9 +43,6 @@ private:
virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override; virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
virtual void visit_edges(Cell::Visitor&) override; virtual void visit_edges(Cell::Visitor&) override;
Optional<FixedArray<Audio::Sample>> get_next_samples();
void play_next_samples();
// https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id
String m_id; String m_id;
@ -66,8 +62,6 @@ private:
JS::GCPtr<AudioTrackList> m_audio_track_list; JS::GCPtr<AudioTrackList> m_audio_track_list;
NonnullOwnPtr<Platform::AudioCodecPlugin> m_audio_plugin; NonnullOwnPtr<Platform::AudioCodecPlugin> m_audio_plugin;
NonnullRefPtr<Audio::Loader> m_loader;
NonnullRefPtr<Platform::Timer> m_sample_timer;
}; };
} }

View file

@ -4,25 +4,51 @@
* SPDX-License-Identifier: BSD-2-Clause * SPDX-License-Identifier: BSD-2-Clause
*/ */
#include <LibAudio/Loader.h>
#include <LibAudio/Resampler.h>
#include <LibAudio/Sample.h>
#include <LibWeb/Platform/AudioCodecPlugin.h> #include <LibWeb/Platform/AudioCodecPlugin.h>
namespace Web::Platform { namespace Web::Platform {
static Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> s_creation_hook; static AudioCodecPlugin::AudioCodecPluginCreator s_creation_hook;
AudioCodecPlugin::AudioCodecPlugin() = default; AudioCodecPlugin::AudioCodecPlugin() = default;
AudioCodecPlugin::~AudioCodecPlugin() = default; AudioCodecPlugin::~AudioCodecPlugin() = default;
void AudioCodecPlugin::install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> creation_hook) void AudioCodecPlugin::install_creation_hook(AudioCodecPluginCreator creation_hook)
{ {
VERIFY(!s_creation_hook); VERIFY(!s_creation_hook);
s_creation_hook = move(creation_hook); s_creation_hook = move(creation_hook);
} }
ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> AudioCodecPlugin::create() ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> AudioCodecPlugin::create(NonnullRefPtr<Audio::Loader> loader)
{ {
VERIFY(s_creation_hook); VERIFY(s_creation_hook);
return s_creation_hook(); return s_creation_hook(move(loader));
}
ErrorOr<FixedArray<Audio::Sample>> AudioCodecPlugin::read_samples_from_loader(Audio::Loader& loader, size_t samples_to_load, size_t device_sample_rate)
{
auto buffer_or_error = loader.get_more_samples(samples_to_load);
if (buffer_or_error.is_error()) {
dbgln("Error while loading samples: {}", buffer_or_error.error().description);
return Error::from_string_literal("Error while loading samples");
}
Audio::ResampleHelper<Audio::Sample> resampler(loader.sample_rate(), device_sample_rate);
return FixedArray<Audio::Sample>::create(resampler.resample(buffer_or_error.release_value()).span());
}
Duration AudioCodecPlugin::current_loader_position(Audio::Loader const& loader, size_t device_sample_rate)
{
auto samples_played = static_cast<double>(loader.loaded_samples());
auto sample_rate = static_cast<double>(loader.sample_rate());
auto source_to_device_ratio = sample_rate / static_cast<double>(device_sample_rate);
samples_played *= source_to_device_ratio;
return Duration::from_milliseconds(static_cast<i64>(samples_played / sample_rate * 1000.0));
} }
} }

View file

@ -9,28 +9,31 @@
#include <AK/FixedArray.h> #include <AK/FixedArray.h>
#include <AK/Function.h> #include <AK/Function.h>
#include <AK/NonnullOwnPtr.h> #include <AK/NonnullOwnPtr.h>
#include <AK/Optional.h> #include <AK/NonnullRefPtr.h>
#include <LibAudio/Forward.h> #include <LibAudio/Forward.h>
namespace Web::Platform { namespace Web::Platform {
class AudioCodecPlugin { class AudioCodecPlugin {
public: public:
static void install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()>); using AudioCodecPluginCreator = Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>(NonnullRefPtr<Audio::Loader>)>;
static ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> create();
static void install_creation_hook(AudioCodecPluginCreator);
static ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> create(NonnullRefPtr<Audio::Loader>);
virtual ~AudioCodecPlugin(); virtual ~AudioCodecPlugin();
virtual size_t device_sample_rate() = 0; static ErrorOr<FixedArray<Audio::Sample>> read_samples_from_loader(Audio::Loader&, size_t samples_to_load, size_t device_sample_rate);
static Duration current_loader_position(Audio::Loader const&, size_t device_sample_rate);
virtual void enqueue_samples(FixedArray<Audio::Sample>) = 0;
virtual size_t remaining_samples() const = 0;
virtual void resume_playback() = 0; virtual void resume_playback() = 0;
virtual void pause_playback() = 0; virtual void pause_playback() = 0;
virtual void playback_ended() = 0;
virtual void set_volume(double) = 0; virtual void set_volume(double) = 0;
virtual void seek(double) = 0;
virtual Duration duration() = 0;
Function<void(Duration)> on_playback_position_updated;
protected: protected:
AudioCodecPlugin(); AudioCodecPlugin();

View file

@ -5,55 +5,82 @@
*/ */
#include <LibAudio/ConnectionToServer.h> #include <LibAudio/ConnectionToServer.h>
#include <LibAudio/Loader.h>
#include <LibAudio/Sample.h>
#include <LibWeb/Platform/Timer.h>
#include <WebContent/AudioCodecPluginSerenity.h> #include <WebContent/AudioCodecPluginSerenity.h>
namespace WebContent { namespace WebContent {
ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> AudioCodecPluginSerenity::create() // These constants and this implementation is based heavily on SoundPlayer::PlaybackManager.
static constexpr u32 UPDATE_RATE_MS = 50;
static constexpr u32 BUFFER_SIZE_MS = 100;
static constexpr size_t ALWAYS_ENQUEUED_BUFFER_COUNT = 5;
ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> AudioCodecPluginSerenity::create(NonnullRefPtr<Audio::Loader> loader)
{ {
auto connection = TRY(Audio::ConnectionToServer::try_create()); auto connection = TRY(Audio::ConnectionToServer::try_create());
return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection))); return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection), move(loader)));
} }
AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer> connection) AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer> connection, NonnullRefPtr<Audio::Loader> loader)
: m_connection(move(connection)) : m_connection(move(connection))
, m_loader(move(loader))
, m_sample_timer(Web::Platform::Timer::create_repeating(UPDATE_RATE_MS, [this]() {
if (play_next_samples().is_error()) {
// FIXME: Propagate the error to the HTMLMediaElement.
} else {
if (on_playback_position_updated)
on_playback_position_updated(m_position);
}
}))
{ {
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
m_duration = Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
m_device_sample_rate = m_connection->get_sample_rate();
m_device_samples_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0 * static_cast<double>(m_device_sample_rate));
m_samples_to_load_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0 * static_cast<double>(m_loader->sample_rate()));
} }
AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default; AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default;
size_t AudioCodecPluginSerenity::device_sample_rate() ErrorOr<void> AudioCodecPluginSerenity::play_next_samples()
{ {
if (!m_device_sample_rate.has_value()) while (m_connection->remaining_samples() < m_device_samples_per_buffer * ALWAYS_ENQUEUED_BUFFER_COUNT) {
m_device_sample_rate = m_connection->get_sample_rate(); bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
return *m_device_sample_rate; bool audio_server_done = m_connection->remaining_samples() == 0;
}
void AudioCodecPluginSerenity::enqueue_samples(FixedArray<Audio::Sample> samples) if (all_samples_loaded && audio_server_done) {
{ pause_playback();
m_connection->async_enqueue(move(samples)).release_value_but_fixme_should_propagate_errors();
}
size_t AudioCodecPluginSerenity::remaining_samples() const m_connection->clear_client_buffer();
{ m_connection->async_clear_buffer();
return m_connection->remaining_samples(); (void)m_loader->reset();
m_position = m_duration;
break;
}
auto samples = TRY(read_samples_from_loader(m_loader, m_samples_to_load_per_buffer, m_device_sample_rate));
TRY(m_connection->async_enqueue(move(samples)));
m_position = current_loader_position(m_loader, m_device_sample_rate);
}
return {};
} }
void AudioCodecPluginSerenity::resume_playback() void AudioCodecPluginSerenity::resume_playback()
{ {
m_connection->async_start_playback(); m_connection->async_start_playback();
m_sample_timer->start();
} }
void AudioCodecPluginSerenity::pause_playback() void AudioCodecPluginSerenity::pause_playback()
{
m_connection->async_start_playback();
}
void AudioCodecPluginSerenity::playback_ended()
{ {
m_connection->async_pause_playback(); m_connection->async_pause_playback();
m_connection->clear_client_buffer(); m_sample_timer->stop();
m_connection->async_clear_buffer();
} }
void AudioCodecPluginSerenity::set_volume(double volume) void AudioCodecPluginSerenity::set_volume(double volume)
@ -61,4 +88,21 @@ void AudioCodecPluginSerenity::set_volume(double volume)
m_connection->async_set_self_volume(volume); m_connection->async_set_self_volume(volume);
} }
void AudioCodecPluginSerenity::seek(double position)
{
auto duration = static_cast<double>(this->duration().to_milliseconds()) / 1000.0;
position = position / duration * static_cast<double>(m_loader->total_samples());
m_loader->seek(static_cast<int>(position)).release_value_but_fixme_should_propagate_errors();
m_position = current_loader_position(m_loader, m_device_sample_rate);
if (on_playback_position_updated)
on_playback_position_updated(m_position);
}
Duration AudioCodecPluginSerenity::duration()
{
return m_duration;
}
} }

View file

@ -7,34 +7,42 @@
#pragma once #pragma once
#include <AK/Error.h> #include <AK/Error.h>
#include <AK/NonnullOwnPtr.h>
#include <AK/NonnullRefPtr.h> #include <AK/NonnullRefPtr.h>
#include <AK/Optional.h> #include <AK/Time.h>
#include <LibAudio/Forward.h> #include <LibAudio/Forward.h>
#include <LibWeb/Forward.h>
#include <LibWeb/Platform/AudioCodecPlugin.h> #include <LibWeb/Platform/AudioCodecPlugin.h>
namespace WebContent { namespace WebContent {
class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin { class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin {
public: public:
static ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> create(); static ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> create(NonnullRefPtr<Audio::Loader>);
virtual ~AudioCodecPluginSerenity() override; virtual ~AudioCodecPluginSerenity() override;
virtual size_t device_sample_rate() override;
virtual void enqueue_samples(FixedArray<Audio::Sample>) override;
virtual size_t remaining_samples() const override;
virtual void resume_playback() override; virtual void resume_playback() override;
virtual void pause_playback() override; virtual void pause_playback() override;
virtual void playback_ended() override;
virtual void set_volume(double) override; virtual void set_volume(double) override;
virtual void seek(double) override;
virtual Duration duration() override;
private: private:
explicit AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer>); AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer>, NonnullRefPtr<Audio::Loader>);
ErrorOr<void> play_next_samples();
NonnullRefPtr<Audio::ConnectionToServer> m_connection; NonnullRefPtr<Audio::ConnectionToServer> m_connection;
Optional<size_t> m_device_sample_rate; NonnullRefPtr<Audio::Loader> m_loader;
NonnullRefPtr<Web::Platform::Timer> m_sample_timer;
Duration m_duration;
Duration m_position;
size_t m_device_sample_rate { 0 };
size_t m_device_samples_per_buffer { 0 };
size_t m_samples_to_load_per_buffer { 0 };
}; };
} }

View file

@ -6,6 +6,7 @@
#include "AudioCodecPluginSerenity.h" #include "AudioCodecPluginSerenity.h"
#include "ImageCodecPluginSerenity.h" #include "ImageCodecPluginSerenity.h"
#include <LibAudio/Loader.h>
#include <LibCore/EventLoop.h> #include <LibCore/EventLoop.h>
#include <LibCore/LocalServer.h> #include <LibCore/LocalServer.h>
#include <LibCore/StandardPaths.h> #include <LibCore/StandardPaths.h>
@ -46,8 +47,8 @@ ErrorOr<int> serenity_main(Main::Arguments)
Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity); Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity);
Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity); Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity);
Web::Platform::AudioCodecPlugin::install_creation_hook([] { Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) {
return WebContent::AudioCodecPluginSerenity::create(); return WebContent::AudioCodecPluginSerenity::create(move(loader));
}); });
Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create())); Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create()));