Userland+LibAudio: Make audio applications support dynamic sample rate

All audio applications (aplay, Piano, Sound Player) respect the ability
of the system to have theoretically any sample rate. Therefore, they
resample their own audio into the system sample rate.

LibAudio previously had its loaders resample their own audio, even
though they expose their sample rate. This is now changed. The loaders
output audio data in their file's sample rate, which the user has to
query and resample appropriately. Resampling code from Buffer, WavLoader
and FlacLoader is removed.

Note that these applications only check the sample rate at startup,
which is reasonable (the user has to restart applications when changing
the sample rate). Fully dynamic adaptation could both lead to errors and
will require another IPC interface. This seems to be enough for now.

Userland/Applications/Piano/AudioPlayerLoop.cpp

@@ -31,12 +31,14 @@ AudioPlayerLoop::AudioPlayerLoop(TrackManager& track_manager, bool& need_to_writ
         (void)buffer_id;
         enqueue_audio();
     };
+    m_resampler = Audio::ResampleHelper<double>(Music::sample_rate, m_audio_client->get_sample_rate());
 }
 
 void AudioPlayerLoop::enqueue_audio()
 {
     m_track_manager.fill_buffer(m_buffer);
     NonnullRefPtr<Audio::Buffer> audio_buffer = music_samples_to_buffer(m_buffer);
+    audio_buffer = Audio::resample_buffer(m_resampler.value(), *audio_buffer);
     m_audio_client->async_enqueue(audio_buffer);
 
     // FIXME: This should be done somewhere else.

Userland/Applications/Piano/AudioPlayerLoop.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "Music.h"
+#include <LibAudio/Buffer.h>
 #include <LibAudio/ClientConnection.h>
 #include <LibAudio/WavWriter.h>
 #include <LibCore/Object.h>
@@ -29,6 +30,7 @@ public:
 private:
     TrackManager& m_track_manager;
     Array<Sample, sample_count> m_buffer;
+    Optional<Audio::ResampleHelper<double>> m_resampler;
     RefPtr<Audio::ClientConnection> m_audio_client;
 
     bool m_should_play_audio = true;

Userland/Applications/Piano/Track.cpp

@@ -127,7 +127,12 @@ String Track::set_recorded_sample(const StringView& path)
     NonnullRefPtr<Audio::Loader> loader = Audio::Loader::create(path);
     if (loader->has_error())
         return String(loader->error_string());
-    auto buffer = loader->get_more_samples(60 * sample_rate * sizeof(Sample)); // 1 minute maximum
+    auto buffer = loader->get_more_samples(60 * loader->sample_rate()); // 1 minute maximum
+    if (loader->has_error())
+        return String(loader->error_string());
+    // Resample to Piano's internal sample rate
+    auto resampler = Audio::ResampleHelper<double>(loader->sample_rate(), sample_rate);
+    buffer = Audio::resample_buffer(resampler, *buffer);
 
     if (!m_recorded_sample.is_empty())
         m_recorded_sample.clear();

Userland/Applications/SoundPlayer/PlaybackManager.cpp

@@ -15,6 +15,7 @@ PlaybackManager::PlaybackManager(NonnullRefPtr<Audio::ClientConnection> connecti
         next_buffer();
     });
     m_timer->stop();
+    m_device_sample_rate = connection->get_sample_rate();
 }
 
 PlaybackManager::~PlaybackManager()
@@ -30,6 +31,7 @@ void PlaybackManager::set_loader(NonnullRefPtr<Audio::Loader>&& loader)
         m_device_samples_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_device_sample_rate;
         u32 source_samples_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_loader->sample_rate();
         m_source_buffer_size_bytes = source_samples_per_buffer * m_loader->num_channels() * m_loader->bits_per_sample() / 8;
+        m_resampler = Audio::ResampleHelper<double>(m_loader->sample_rate(), m_device_sample_rate);
         m_timer->start();
     } else {
         m_timer->stop();
@@ -116,6 +118,9 @@ void PlaybackManager::next_buffer()
 
     if (audio_server_remaining_samples < m_device_samples_per_buffer) {
         m_current_buffer = m_loader->get_more_samples(m_source_buffer_size_bytes);
+        VERIFY(m_resampler.has_value());
+        m_resampler->reset();
+        m_current_buffer = Audio::resample_buffer(m_resampler.value(), *m_current_buffer);
         if (m_current_buffer)
             m_connection->enqueue(*m_current_buffer);
     }

Userland/Applications/SoundPlayer/PlaybackManager.h

@@ -52,6 +52,7 @@ private:
     RefPtr<Audio::Loader> m_loader { nullptr };
     NonnullRefPtr<Audio::ClientConnection> m_connection;
     RefPtr<Audio::Buffer> m_current_buffer;
+    Optional<Audio::ResampleHelper<double>> m_resampler;
     RefPtr<Core::Timer> m_timer;
 
     // Controls the GUI update rate. A smaller value makes the visualizations nicer.

Userland/Libraries/LibAudio/Buffer.cpp

@@ -45,7 +45,7 @@ i32 Buffer::allocate_id()
 }
 
 template<typename SampleReader>
-static void read_samples_from_stream(InputMemoryStream& stream, SampleReader read_sample, Vector<Frame>& samples, ResampleHelper<double>& resampler, int num_channels)
+static void read_samples_from_stream(InputMemoryStream& stream, SampleReader read_sample, Vector<Frame>& samples, int num_channels)
 {
     double norm_l = 0;
     double norm_r = 0;
@@ -53,29 +53,23 @@ static void read_samples_from_stream(InputMemoryStream& stream, SampleReader rea
     switch (num_channels) {
     case 1:
         for (;;) {
-            while (resampler.read_sample(norm_l, norm_r)) {
-                samples.append(Frame(norm_l));
-            }
             norm_l = read_sample(stream);
+            samples.append(Frame(norm_l));
 
             if (stream.handle_any_error()) {
                 break;
             }
-            resampler.process_sample(norm_l, norm_r);
         }
         break;
     case 2:
         for (;;) {
-            while (resampler.read_sample(norm_l, norm_r)) {
-                samples.append(Frame(norm_l, norm_r));
-            }
             norm_l = read_sample(stream);
             norm_r = read_sample(stream);
+            samples.append(Frame(norm_l, norm_r));
 
             if (stream.handle_any_error()) {
                 break;
             }
-            resampler.process_sample(norm_l, norm_r);
         }
         break;
     default:
@@ -128,32 +122,32 @@ static double read_norm_sample_8(InputMemoryStream& stream)
     return double(sample) / NumericLimits<u8>::max();
 }
 
-RefPtr<Buffer> Buffer::from_pcm_data(ReadonlyBytes data, ResampleHelper<double>& resampler, int num_channels, PcmSampleFormat sample_format)
+RefPtr<Buffer> Buffer::from_pcm_data(ReadonlyBytes data, int num_channels, PcmSampleFormat sample_format)
 {
     InputMemoryStream stream { data };
-    return from_pcm_stream(stream, resampler, num_channels, sample_format, data.size() / (pcm_bits_per_sample(sample_format) / 8));
+    return from_pcm_stream(stream, num_channels, sample_format, data.size() / (pcm_bits_per_sample(sample_format) / 8));
 }
 
-RefPtr<Buffer> Buffer::from_pcm_stream(InputMemoryStream& stream, ResampleHelper<double>& resampler, int num_channels, PcmSampleFormat sample_format, int num_samples)
+RefPtr<Buffer> Buffer::from_pcm_stream(InputMemoryStream& stream, int num_channels, PcmSampleFormat sample_format, int num_samples)
 {
     Vector<Frame> fdata;
     fdata.ensure_capacity(num_samples);
 
     switch (sample_format) {
     case PcmSampleFormat::Uint8:
-        read_samples_from_stream(stream, read_norm_sample_8, fdata, resampler, num_channels);
+        read_samples_from_stream(stream, read_norm_sample_8, fdata, num_channels);
         break;
     case PcmSampleFormat::Int16:
-        read_samples_from_stream(stream, read_norm_sample_16, fdata, resampler, num_channels);
+        read_samples_from_stream(stream, read_norm_sample_16, fdata, num_channels);
         break;
     case PcmSampleFormat::Int24:
-        read_samples_from_stream(stream, read_norm_sample_24, fdata, resampler, num_channels);
+        read_samples_from_stream(stream, read_norm_sample_24, fdata, num_channels);
         break;
     case PcmSampleFormat::Float32:
-        read_samples_from_stream(stream, read_float_sample_32, fdata, resampler, num_channels);
+        read_samples_from_stream(stream, read_float_sample_32, fdata, num_channels);
         break;
     case PcmSampleFormat::Float64:
-        read_samples_from_stream(stream, read_float_sample_64, fdata, resampler, num_channels);
+        read_samples_from_stream(stream, read_float_sample_64, fdata, num_channels);
         break;
     default:
         VERIFY_NOT_REACHED();
@@ -193,6 +187,21 @@ Vector<SampleType> ResampleHelper<SampleType>::resample(Vector<SampleType> to_re
 template Vector<i32> ResampleHelper<i32>::resample(Vector<i32>);
 template Vector<double> ResampleHelper<double>::resample(Vector<double>);
 
+NonnullRefPtr<Buffer> resample_buffer(ResampleHelper<double>& resampler, Buffer const& to_resample)
+{
+    Vector<Frame> resampled;
+    resampled.ensure_capacity(to_resample.sample_count() * ceil_div(resampler.source(), resampler.target()));
+    for (size_t i = 0; i < static_cast<size_t>(to_resample.sample_count()); ++i) {
+        auto sample = to_resample.samples()[i];
+        resampler.process_sample(sample.left, sample.right);
+
+        while (resampler.read_sample(sample.left, sample.right))
+            resampled.append(sample);
+    }
+
+    return Buffer::create_with_samples(move(resampled));
+}
+
 template<typename SampleType>
 void ResampleHelper<SampleType>::process_sample(SampleType sample_l, SampleType sample_r)
 {

Userland/Libraries/LibAudio/Buffer.h

@@ -105,6 +105,9 @@ public:
 
     void reset();
 
+    u32 source() const { return m_source; }
+    u32 target() const { return m_target; }
+
 private:
     const u32 m_source;
     const u32 m_target;
@@ -113,11 +116,11 @@ private:
     SampleType m_last_sample_r;
 };
 
-// A buffer of audio samples, normalized to 44100hz.
+// A buffer of audio samples.
 class Buffer : public RefCounted<Buffer> {
 public:
-    static RefPtr<Buffer> from_pcm_data(ReadonlyBytes data, ResampleHelper<double>& resampler, int num_channels, PcmSampleFormat sample_format);
-    static RefPtr<Buffer> from_pcm_stream(InputMemoryStream& stream, ResampleHelper<double>& resampler, int num_channels, PcmSampleFormat sample_format, int num_samples);
+    static RefPtr<Buffer> from_pcm_data(ReadonlyBytes data, int num_channels, PcmSampleFormat sample_format);
+    static RefPtr<Buffer> from_pcm_stream(InputMemoryStream& stream, int num_channels, PcmSampleFormat sample_format, int num_samples);
     static NonnullRefPtr<Buffer> create_with_samples(Vector<Frame>&& samples)
     {
         return adopt_ref(*new Buffer(move(samples)));
@@ -157,4 +160,7 @@ private:
     const int m_sample_count;
 };
 
+// This only works for double resamplers, and therefore cannot be part of the class
+NonnullRefPtr<Buffer> resample_buffer(ResampleHelper<double>& resampler, Buffer const& to_resample);
+
 }

Userland/Libraries/LibAudio/FlacLoader.cpp

@@ -40,8 +40,6 @@ FlacLoaderPlugin::FlacLoaderPlugin(const StringView& path)
     reset();
     if (!m_valid)
         return;
-
-    m_resampler = make<ResampleHelper<i32>>(m_sample_rate, 44100);
 }
 
 FlacLoaderPlugin::FlacLoaderPlugin(const ByteBuffer& buffer)
@@ -58,8 +56,6 @@ FlacLoaderPlugin::FlacLoaderPlugin(const ByteBuffer& buffer)
     reset();
     if (!m_valid)
         return;
-
-    m_resampler = make<ResampleHelper<i32>>(m_sample_rate, 44100);
 }
 
 bool FlacLoaderPlugin::sniff()
@@ -348,8 +344,6 @@ void FlacLoaderPlugin::next_frame()
         FlacSubframeHeader new_subframe = next_subframe_header(bit_stream, i);
         CHECK_ERROR_STRING;
         Vector<i32> subframe_samples = parse_subframe(new_subframe, bit_stream);
-        m_resampler->reset();
-        subframe_samples = m_resampler->resample(subframe_samples);
         CHECK_ERROR_STRING;
         current_subframes.append(move(subframe_samples));
     }

Userland/Libraries/LibAudio/FlacLoader.h

@@ -124,7 +124,6 @@ private:
     bool m_valid { false };
     RefPtr<Core::File> m_file;
     String m_error_string;
-    OwnPtr<ResampleHelper<i32>> m_resampler;
 
     // Data obtained directly from the FLAC metadata: many values have specific bit counts
     u32 m_sample_rate { 0 };         // 20 bit

Userland/Libraries/LibAudio/WavLoader.cpp

@@ -29,8 +29,6 @@ WavLoaderPlugin::WavLoaderPlugin(const StringView& path)
     valid = parse_header();
     if (!valid)
         return;
-
-    m_resampler = make<ResampleHelper<double>>(m_sample_rate, m_device_sample_rate);
 }
 
 WavLoaderPlugin::WavLoaderPlugin(const ByteBuffer& buffer)
@@ -45,8 +43,6 @@ WavLoaderPlugin::WavLoaderPlugin(const ByteBuffer& buffer)
     valid = parse_header();
     if (!valid)
         return;
-
-    m_resampler = make<ResampleHelper<double>>(m_sample_rate, m_device_sample_rate);
 }
 
 RefPtr<Buffer> WavLoaderPlugin::get_more_samples(size_t max_bytes_to_read_from_input)
@@ -81,7 +77,6 @@ RefPtr<Buffer> WavLoaderPlugin::get_more_samples(size_t max_bytes_to_read_from_i
 
     RefPtr<Buffer> buffer = Buffer::from_pcm_data(
         sample_data.bytes(),
-        *m_resampler,
         m_num_channels,
         m_sample_format);
 

Userland/Libraries/LibAudio/WavLoader.h

@@ -67,19 +67,11 @@ private:
     AK::InputMemoryStream* m_memory_stream;
     String m_error_string;
 
-    // TODO: We should probably move resampling into the audio server.
-    //
-    // It would avoid duplicate resampling code and would allow clients
-    // to be agnostic of the destination audio device's sample rate.
-    OwnPtr<ResampleHelper<double>> m_resampler;
-
     u32 m_sample_rate { 0 };
     u16 m_num_channels { 0 };
     PcmSampleFormat m_sample_format;
     size_t m_byte_offset_of_data_samples { 0 };
 
-    // FIXME: Get this value from the audio server
-    int m_device_sample_rate { 44100 };
     int m_loaded_samples { 0 };
     int m_total_samples { 0 };
 };

Userland/Utilities/aplay.cpp

@@ -35,12 +35,17 @@ int main(int argc, char** argv)
         loader->bits_per_sample(),
         loader->num_channels() == 1 ? "Mono" : "Stereo");
     out("\033[34;1mProgress\033[0m: \033[s");
+
+    auto resampler = Audio::ResampleHelper<double>(loader->sample_rate(), audio_client->get_sample_rate());
+
     for (;;) {
         auto samples = loader->get_more_samples();
         if (samples) {
             out("\033[u");
             out("{}/{}", loader->loaded_samples(), loader->total_samples());
             fflush(stdout);
+            resampler.reset();
+            samples = Audio::resample_buffer(resampler, *samples);
             audio_client->enqueue(*samples);
         } else if (loader->has_error()) {
             outln();