From 57783eff246a0e74fc5daa1fd954c220491df374 Mon Sep 17 00:00:00 2001 From: Jelle Raaijmakers Date: Mon, 30 Sep 2024 14:33:08 +0200 Subject: [PATCH] LibMedia: Convert OggLoader into generic FFmpegLoader This loader supports whatever format libavformat and libavcodec can handle. Currently only seekable streams are supported, and we still have some limitations as to the number of channels and sample format. Plays all non-streaming audio files at: https://tools.woolyss.com/html5-audio-video-tester/ --- .github/actions/setup/action.yml | 4 +- Documentation/BuildInstructionsLadybird.md | 2 +- Meta/CMake/ffmpeg.cmake | 3 +- .../Userland/Libraries/LibMedia/BUILD.gn | 2 +- Tests/LibMedia/TestVorbisDecode.cpp | 2 +- Tests/LibMedia/TestWav.cpp | 4 +- .../Libraries/LibMedia/Audio/FFmpegLoader.cpp | 345 ++++++++++++++++++ .../Audio/{OggLoader.h => FFmpegLoader.h} | 34 +- Userland/Libraries/LibMedia/Audio/Loader.cpp | 4 +- .../Libraries/LibMedia/Audio/OggLoader.cpp | 266 -------------- Userland/Libraries/LibMedia/CMakeLists.txt | 4 +- 11 files changed, 382 insertions(+), 288 deletions(-) create mode 100644 Userland/Libraries/LibMedia/Audio/FFmpegLoader.cpp rename Userland/Libraries/LibMedia/Audio/{OggLoader.h => FFmpegLoader.h} (63%) delete mode 100644 Userland/Libraries/LibMedia/Audio/OggLoader.cpp diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index ff3cf777ca9..dda8eb27f27 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,8 +26,8 @@ runs: sudo apt-get update sudo apt-get install autoconf autoconf-archive automake build-essential ccache clang-18 clang++-18 cmake curl fonts-liberation2 \ - gcc-13 g++-13 libavcodec-dev libavformat-dev libegl1-mesa-dev libgl1-mesa-dev libpulse-dev libssl-dev libstdc++-13-dev lld-18 \ - nasm ninja-build qt6-base-dev qt6-tools-dev-tools tar unzip zip + gcc-13 g++-13 libavcodec-dev libavformat-dev libavutil-dev libegl1-mesa-dev libgl1-mesa-dev libpulse-dev libssl-dev \ + libstdc++-13-dev lld-18 nasm ninja-build qt6-base-dev qt6-tools-dev-tools tar unzip zip sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 diff --git a/Documentation/BuildInstructionsLadybird.md b/Documentation/BuildInstructionsLadybird.md index acf01ddb3e2..7bc12bf0585 100644 --- a/Documentation/BuildInstructionsLadybird.md +++ b/Documentation/BuildInstructionsLadybird.md @@ -14,7 +14,7 @@ CMake 3.25 or newer must be available in $PATH. ### Debian/Ubuntu: ```bash -sudo apt install autoconf autoconf-archive automake build-essential ccache cmake curl fonts-liberation2 git libavcodec-dev libavformat-dev libgl1-mesa-dev nasm ninja-build pkg-config qt6-base-dev qt6-tools-dev-tools qt6-wayland tar unzip zip +sudo apt install autoconf autoconf-archive automake build-essential ccache cmake curl fonts-liberation2 git libavcodec-dev libavformat-dev libavutil-dev libgl1-mesa-dev nasm ninja-build pkg-config qt6-base-dev qt6-tools-dev-tools qt6-wayland tar unzip zip ``` #### CMake 3.25 or newer: diff --git a/Meta/CMake/ffmpeg.cmake b/Meta/CMake/ffmpeg.cmake index dfc9b5c890d..0b5b8711cbe 100644 --- a/Meta/CMake/ffmpeg.cmake +++ b/Meta/CMake/ffmpeg.cmake @@ -3,8 +3,9 @@ include_guard() find_package(PkgConfig REQUIRED) pkg_check_modules(AVCODEC IMPORTED_TARGET libavcodec) pkg_check_modules(AVFORMAT IMPORTED_TARGET libavformat) +pkg_check_modules(AVUTIL IMPORTED_TARGET libavutil) -if (AVCODEC_FOUND AND AVFORMAT_FOUND) +if (AVCODEC_FOUND AND AVFORMAT_FOUND AND AVUTIL_FOUND) set(HAS_FFMPEG ON CACHE BOOL "" FORCE) add_compile_definitions(USE_FFMPEG=1) endif() diff --git a/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn b/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn index 9e31053ea2e..cca68e8499f 100644 --- a/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn +++ b/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn @@ -31,7 +31,7 @@ shared_library("LibMedia") { } if (enable_ffmpeg) { sources += [ - "Audio/OggLoader.cpp", + "Audio/FFmpegLoader.cpp", "FFmpeg/FFmpegVideoDecoder.cpp", ] } else { diff --git a/Tests/LibMedia/TestVorbisDecode.cpp b/Tests/LibMedia/TestVorbisDecode.cpp index 92a6c82a07c..e062c27d0fd 100644 --- a/Tests/LibMedia/TestVorbisDecode.cpp +++ b/Tests/LibMedia/TestVorbisDecode.cpp @@ -9,7 +9,7 @@ static void run_test(StringView file_name, int const num_samples, int const channels, u32 const rate) { - constexpr auto format = "Ogg Vorbis (.ogg)"; + constexpr auto format = "ogg"; constexpr int bits = 32; ByteString in_path = ByteString::formatted("vorbis/{}", file_name); diff --git a/Tests/LibMedia/TestWav.cpp b/Tests/LibMedia/TestWav.cpp index f5896acc1ad..f4b0037533c 100644 --- a/Tests/LibMedia/TestWav.cpp +++ b/Tests/LibMedia/TestWav.cpp @@ -10,8 +10,8 @@ static void run_test(StringView file_name, int const num_samples, int const channels, u32 const rate) { - constexpr auto format = "RIFF WAVE (.wav)"; - constexpr int bits = 16; + constexpr auto format = "wav"; + constexpr int bits = 32; ByteString in_path = ByteString::formatted("WAV/{}", file_name); diff --git a/Userland/Libraries/LibMedia/Audio/FFmpegLoader.cpp b/Userland/Libraries/LibMedia/Audio/FFmpegLoader.cpp new file mode 100644 index 00000000000..ffc21eba60b --- /dev/null +++ b/Userland/Libraries/LibMedia/Audio/FFmpegLoader.cpp @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2024, Jelle Raaijmakers + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "FFmpegLoader.h" +#include +#include +#include + +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59, 24, 100) +# define USE_FFMPEG_CH_LAYOUT +#endif +#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(59, 0, 100) +# define USE_CONSTIFIED_POINTERS +#endif + +namespace Audio { + +static constexpr int BUFFER_MAX_PROBE_SIZE = 64 * KiB; + +FFmpegIOContext::FFmpegIOContext(AVIOContext* avio_context) + : m_avio_context(avio_context) +{ +} + +FFmpegIOContext::~FFmpegIOContext() +{ + // NOTE: free the buffer inside the AVIO context, since it might be changed since its initial allocation + av_free(m_avio_context->buffer); + avio_context_free(&m_avio_context); +} + +ErrorOr, LoaderError> FFmpegIOContext::create(AK::SeekableStream& stream) +{ + auto* avio_buffer = av_malloc(PAGE_SIZE); + if (avio_buffer == nullptr) + return LoaderError { LoaderError::Category::IO, "Failed to allocate AVIO buffer" }; + + // This AVIOContext explains to avformat how to interact with our stream + auto* avio_context = avio_alloc_context( + static_cast(avio_buffer), + PAGE_SIZE, + 0, + &stream, + [](void* opaque, u8* buffer, int size) -> int { + auto& stream = *static_cast(opaque); + AK::Bytes buffer_bytes { buffer, AK::min(size, PAGE_SIZE) }; + auto read_bytes_or_error = stream.read_some(buffer_bytes); + if (read_bytes_or_error.is_error()) { + if (read_bytes_or_error.error().code() == EOF) + return AVERROR_EOF; + return AVERROR_UNKNOWN; + } + int number_of_bytes_read = read_bytes_or_error.value().size(); + if (number_of_bytes_read == 0) + return AVERROR_EOF; + return number_of_bytes_read; + }, + nullptr, + [](void* opaque, int64_t offset, int whence) -> int64_t { + whence &= ~AVSEEK_FORCE; + + auto& stream = *static_cast(opaque); + if (whence == AVSEEK_SIZE) + return static_cast(stream.size().value()); + + auto seek_mode_from_whence = [](int origin) -> SeekMode { + if (origin == SEEK_CUR) + return SeekMode::FromCurrentPosition; + if (origin == SEEK_END) + return SeekMode::FromEndPosition; + return SeekMode::SetPosition; + }; + auto offset_or_error = stream.seek(offset, seek_mode_from_whence(whence)); + if (offset_or_error.is_error()) + return -EIO; + return 0; + }); + if (avio_context == nullptr) { + av_free(avio_buffer); + return LoaderError { LoaderError::Category::IO, "Failed to allocate AVIO context" }; + } + + return make(avio_context); +} + +FFmpegLoaderPlugin::FFmpegLoaderPlugin(NonnullOwnPtr stream, NonnullOwnPtr io_context) + : LoaderPlugin(move(stream)) + , m_io_context(move(io_context)) +{ +} + +FFmpegLoaderPlugin::~FFmpegLoaderPlugin() +{ + if (m_frame != nullptr) + av_frame_free(&m_frame); + if (m_packet != nullptr) + av_packet_free(&m_packet); + if (m_codec_context != nullptr) + avcodec_free_context(&m_codec_context); + if (m_format_context != nullptr) + avformat_close_input(&m_format_context); +} + +ErrorOr, LoaderError> FFmpegLoaderPlugin::create(NonnullOwnPtr stream) +{ + auto io_context = TRY(FFmpegIOContext::create(*stream)); + auto loader = make(move(stream), move(io_context)); + TRY(loader->initialize()); + return loader; +} + +MaybeLoaderError FFmpegLoaderPlugin::initialize() +{ + // Open the container + m_format_context = avformat_alloc_context(); + if (m_format_context == nullptr) + return LoaderError { LoaderError::Category::IO, "Failed to allocate format context" }; + m_format_context->pb = m_io_context->avio_context(); + if (avformat_open_input(&m_format_context, nullptr, nullptr, nullptr) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to open input for format parsing" }; + + // Read stream info; doing this is required for headerless formats like MPEG + if (avformat_find_stream_info(m_format_context, nullptr) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to find stream info" }; + +#ifdef USE_CONSTIFIED_POINTERS + AVCodec const* codec {}; +#else + AVCodec* codec {}; +#endif + // Find the best stream to play within the container + int best_stream_index = av_find_best_stream(m_format_context, AVMediaType::AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0); + if (best_stream_index == AVERROR_STREAM_NOT_FOUND) + return LoaderError { LoaderError::Category::Format, "No audio stream found in container" }; + if (best_stream_index == AVERROR_DECODER_NOT_FOUND) + return LoaderError { LoaderError::Category::Format, "No suitable decoder found for stream" }; + if (best_stream_index < 0) + return LoaderError { LoaderError::Category::Format, "Failed to find an audio stream" }; + m_audio_stream = m_format_context->streams[best_stream_index]; + + // Set up the context to decode the audio stream + m_codec_context = avcodec_alloc_context3(codec); + if (m_codec_context == nullptr) + return LoaderError { LoaderError::Category::IO, "Failed to allocate the codec context" }; + + if (avcodec_parameters_to_context(m_codec_context, m_audio_stream->codecpar) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to copy codec parameters" }; + + m_codec_context->pkt_timebase = m_audio_stream->time_base; + m_codec_context->thread_count = AK::min(static_cast(Core::System::hardware_concurrency()), 4); + + if (avcodec_open2(m_codec_context, codec, nullptr) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to open input for decoding" }; + + // This is an initial estimate of the total number of samples in the stream. + // During decoding, we might need to increase the number as more frames come in. + double duration_in_seconds = static_cast(m_audio_stream->duration) * time_base(); + if (duration_in_seconds < 0) + return LoaderError { LoaderError::Category::Format, "Negative stream duration" }; + m_total_samples = AK::round_to(sample_rate() * duration_in_seconds); + + // Allocate packet (logical chunk of data) and frame (video / audio frame) buffers + m_packet = av_packet_alloc(); + if (m_packet == nullptr) + return LoaderError { LoaderError::Category::IO, "Failed to allocate packet" }; + + m_frame = av_frame_alloc(); + if (m_frame == nullptr) + return LoaderError { LoaderError::Category::IO, "Failed to allocate frame" }; + + return {}; +} + +double FFmpegLoaderPlugin::time_base() const +{ + return av_q2d(m_audio_stream->time_base); +} + +bool FFmpegLoaderPlugin::sniff(SeekableStream& stream) +{ + auto io_context = MUST(FFmpegIOContext::create(stream)); +#ifdef USE_CONSTIFIED_POINTERS + AVInputFormat const* detected_format {}; +#else + AVInputFormat* detected_format {}; +#endif + auto score = av_probe_input_buffer2(io_context->avio_context(), &detected_format, nullptr, nullptr, 0, BUFFER_MAX_PROBE_SIZE); + return score > 0; +} + +static ErrorOr> extract_samples_from_frame(AVFrame& frame) +{ + size_t number_of_samples = frame.nb_samples; + VERIFY(number_of_samples > 0); + +#ifdef USE_FFMPEG_CH_LAYOUT + size_t number_of_channels = frame.ch_layout.nb_channels; +#else + size_t number_of_channels = frame.channels; +#endif + auto format = static_cast(frame.format); + auto packed_format = av_get_packed_sample_fmt(format); + auto is_planar = av_sample_fmt_is_planar(format) == 1; + + // FIXME: handle number_of_channels > 2 + if (number_of_channels != 1 && number_of_channels != 2) + return Error::from_string_view("Unsupported number of channels"sv); + + switch (format) { + case AV_SAMPLE_FMT_FLTP: + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S32: + break; + default: + // FIXME: handle other formats + return Error::from_string_view("Unsupported sample format"sv); + } + + auto get_plane_pointer = [&](size_t channel_index) -> uint8_t* { + return is_planar ? frame.extended_data[channel_index] : frame.extended_data[0]; + }; + auto index_in_plane = [&](size_t sample_index, size_t channel_index) { + if (is_planar) + return sample_index; + return sample_index * number_of_channels + channel_index; + }; + auto read_sample = [&](uint8_t* data, size_t index) -> float { + switch (packed_format) { + case AV_SAMPLE_FMT_FLT: + return reinterpret_cast(data)[index]; + case AV_SAMPLE_FMT_S16: + return reinterpret_cast(data)[index] / static_cast(NumericLimits::max()); + case AV_SAMPLE_FMT_S32: + return reinterpret_cast(data)[index] / static_cast(NumericLimits::max()); + default: + VERIFY_NOT_REACHED(); + } + }; + + auto samples = TRY(FixedArray::create(number_of_samples)); + for (size_t sample = 0; sample < number_of_samples; ++sample) { + if (number_of_channels == 1) { + samples.unchecked_at(sample) = Sample { read_sample(get_plane_pointer(0), index_in_plane(sample, 0)) }; + } else { + samples.unchecked_at(sample) = Sample { + read_sample(get_plane_pointer(0), index_in_plane(sample, 0)), + read_sample(get_plane_pointer(1), index_in_plane(sample, 1)), + }; + } + } + return samples; +} + +ErrorOr>, LoaderError> FFmpegLoaderPlugin::load_chunks(size_t samples_to_read_from_input) +{ + Vector> chunks {}; + + do { + // Obtain a packet + if (av_read_frame(m_format_context, m_packet) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to read frame" }; + if (m_packet->stream_index != m_audio_stream->index) { + av_packet_unref(m_packet); + continue; + } + + // Send the packet to the decoder + if (avcodec_send_packet(m_codec_context, m_packet) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to send packet" }; + av_packet_unref(m_packet); + + // Ask the decoder for a new frame. We might not have sent enough data yet + auto receive_frame_error = avcodec_receive_frame(m_codec_context, m_frame); + if (receive_frame_error != 0) { + if (receive_frame_error == AVERROR(EAGAIN)) + continue; + if (receive_frame_error == AVERROR_EOF) + return Error::from_errno(EOF); + return LoaderError { LoaderError::Category::IO, "Failed to receive frame" }; + } + + chunks.append(TRY(extract_samples_from_frame(*m_frame))); + + // Use the frame's presentation timestamp to set the number of loaded samples + m_loaded_samples = static_cast(m_frame->pts * sample_rate() * time_base()); + if (m_loaded_samples > m_total_samples) [[unlikely]] + m_total_samples = m_loaded_samples; + + samples_to_read_from_input -= AK::min(samples_to_read_from_input, m_frame->nb_samples); + } while (samples_to_read_from_input > 0); + + return chunks; +} + +MaybeLoaderError FFmpegLoaderPlugin::reset() +{ + return seek(0); +} + +MaybeLoaderError FFmpegLoaderPlugin::seek(int sample_index) +{ + auto sample_position_in_seconds = static_cast(sample_index) / sample_rate(); + auto sample_timestamp = AK::round_to(sample_position_in_seconds / time_base()); + + if (av_seek_frame(m_format_context, m_audio_stream->index, sample_timestamp, AVSEEK_FLAG_ANY) < 0) + return LoaderError { LoaderError::Category::IO, "Failed to seek" }; + avcodec_flush_buffers(m_codec_context); + + m_loaded_samples = sample_index; + return {}; +} + +u32 FFmpegLoaderPlugin::sample_rate() +{ + VERIFY(m_codec_context != nullptr); + return m_codec_context->sample_rate; +} + +u16 FFmpegLoaderPlugin::num_channels() +{ + VERIFY(m_codec_context != nullptr); +#ifdef USE_FFMPEG_CH_LAYOUT + return m_codec_context->ch_layout.nb_channels; +#else + return m_codec_context->channels; +#endif +} + +PcmSampleFormat FFmpegLoaderPlugin::pcm_format() +{ + // FIXME: pcm_format() is unused, always return Float for now + return PcmSampleFormat::Float32; +} + +ByteString FFmpegLoaderPlugin::format_name() +{ + if (!m_format_context) + return "unknown"; + return m_format_context->iformat->name; +} + +} diff --git a/Userland/Libraries/LibMedia/Audio/OggLoader.h b/Userland/Libraries/LibMedia/Audio/FFmpegLoader.h similarity index 63% rename from Userland/Libraries/LibMedia/Audio/OggLoader.h rename to Userland/Libraries/LibMedia/Audio/FFmpegLoader.h index 0a965731754..684eceea4b5 100644 --- a/Userland/Libraries/LibMedia/Audio/OggLoader.h +++ b/Userland/Libraries/LibMedia/Audio/FFmpegLoader.h @@ -7,18 +7,34 @@ #pragma once #include "Loader.h" +#include +#include extern "C" { #include #include +#include } namespace Audio { -class OggLoaderPlugin : public LoaderPlugin { +class FFmpegIOContext { public: - explicit OggLoaderPlugin(NonnullOwnPtr stream); - virtual ~OggLoaderPlugin(); + explicit FFmpegIOContext(AVIOContext*); + ~FFmpegIOContext(); + + static ErrorOr, LoaderError> create(AK::SeekableStream& stream); + + AVIOContext* avio_context() const { return m_avio_context; } + +private: + AVIOContext* m_avio_context { nullptr }; +}; + +class FFmpegLoaderPlugin : public LoaderPlugin { +public: + explicit FFmpegLoaderPlugin(NonnullOwnPtr, NonnullOwnPtr); + virtual ~FFmpegLoaderPlugin(); static bool sniff(SeekableStream& stream); static ErrorOr, LoaderError> create(NonnullOwnPtr); @@ -33,21 +49,19 @@ public: virtual u32 sample_rate() override; virtual u16 num_channels() override; virtual PcmSampleFormat pcm_format() override; - virtual ByteString format_name() override { return "Ogg Vorbis (.ogg)"; } + virtual ByteString format_name() override; private: MaybeLoaderError initialize(); double time_base() const; - void* m_avio_buffer { nullptr }; - AVIOContext* m_avio_context { nullptr }; + AVStream* m_audio_stream; AVCodecContext* m_codec_context { nullptr }; AVFormatContext* m_format_context { nullptr }; - AVStream* m_audio_stream; - AVFrame* m_frame; - AVPacket* m_packet; - + AVFrame* m_frame { nullptr }; + NonnullOwnPtr m_io_context; int m_loaded_samples { 0 }; + AVPacket* m_packet { nullptr }; int m_total_samples { 0 }; }; diff --git a/Userland/Libraries/LibMedia/Audio/Loader.cpp b/Userland/Libraries/LibMedia/Audio/Loader.cpp index 7f0bed4ee10..dec5d53fe4c 100644 --- a/Userland/Libraries/LibMedia/Audio/Loader.cpp +++ b/Userland/Libraries/LibMedia/Audio/Loader.cpp @@ -6,9 +6,9 @@ */ #include "Loader.h" +#include "FFmpegLoader.h" #include "FlacLoader.h" #include "MP3Loader.h" -#include "OggLoader.h" #include "QOALoader.h" #include "WavLoader.h" #include @@ -35,7 +35,7 @@ static constexpr LoaderPluginInitializer s_initializers[] = { { FlacLoaderPlugin::sniff, FlacLoaderPlugin::create }, { QOALoaderPlugin::sniff, QOALoaderPlugin::create }, #ifdef USE_FFMPEG - { OggLoaderPlugin::sniff, OggLoaderPlugin::create }, + { FFmpegLoaderPlugin::sniff, FFmpegLoaderPlugin::create }, #endif { WavLoaderPlugin::sniff, WavLoaderPlugin::create }, { MP3LoaderPlugin::sniff, MP3LoaderPlugin::create }, diff --git a/Userland/Libraries/LibMedia/Audio/OggLoader.cpp b/Userland/Libraries/LibMedia/Audio/OggLoader.cpp deleted file mode 100644 index 178cf82a323..00000000000 --- a/Userland/Libraries/LibMedia/Audio/OggLoader.cpp +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright (c) 2024, Jelle Raaijmakers - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "OggLoader.h" -#include -#include -#include - -#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59, 24, 100) -# define USE_FFMPEG_CH_LAYOUT -#endif - -namespace Audio { - -OggLoaderPlugin::OggLoaderPlugin(NonnullOwnPtr stream) - : LoaderPlugin(move(stream)) -{ -} - -OggLoaderPlugin::~OggLoaderPlugin() -{ - av_frame_free(&m_frame); - av_packet_free(&m_packet); - avcodec_free_context(&m_codec_context); - avformat_close_input(&m_format_context); - avio_context_free(&m_avio_context); - av_free(m_avio_buffer); -} - -ErrorOr, LoaderError> OggLoaderPlugin::create(NonnullOwnPtr stream) -{ - auto loader = make(move(stream)); - TRY(loader->initialize()); - return loader; -} - -MaybeLoaderError OggLoaderPlugin::initialize() -{ - m_format_context = avformat_alloc_context(); - if (m_format_context == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to allocate format context" }; - - m_avio_buffer = av_malloc(PAGE_SIZE); - if (m_avio_buffer == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to allocate AVIO buffer" }; - - // This AVIOContext explains to avformat how to interact with our stream - m_avio_context = avio_alloc_context( - static_cast(m_avio_buffer), - PAGE_SIZE, - 0, - m_stream.ptr(), - [](void* opaque, u8* buffer, int size) -> int { - auto& stream = *static_cast(opaque); - AK::Bytes buffer_bytes { buffer, static_cast(size) }; - auto read_bytes_or_error = stream.read_some(buffer_bytes); - if (read_bytes_or_error.is_error()) { - if (read_bytes_or_error.error().code() == EOF) - return AVERROR_EOF; - return AVERROR_UNKNOWN; - } - return static_cast(read_bytes_or_error.value().size()); - }, - nullptr, - [](void* opaque, int64_t offset, int origin) -> int64_t { - auto& stream = *static_cast(opaque); - auto seek_mode_from_whence = [](int origin) -> SeekMode { - if (origin == SEEK_CUR) - return SeekMode::FromCurrentPosition; - if (origin == SEEK_END) - return SeekMode::FromEndPosition; - return SeekMode::SetPosition; - }; - auto offset_or_error = stream.seek(offset, seek_mode_from_whence(origin)); - if (offset_or_error.is_error()) - return -EIO; - return 0; - }); - - m_format_context->pb = m_avio_context; - - // Open the stream as an ogg container - auto* av_input_format = av_find_input_format("ogg"); - if (av_input_format == nullptr) - return LoaderError { LoaderError::Category::Internal, "Failed to obtain input format" }; - - if (avformat_open_input(&m_format_context, nullptr, av_input_format, nullptr) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to open input for format parsing" }; - - // Find the best stream to play within the container - int best_stream_index = av_find_best_stream(m_format_context, AVMediaType::AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); - if (best_stream_index < 0) - return LoaderError { LoaderError::Category::Format, "Failed to find an audio stream" }; - m_audio_stream = m_format_context->streams[best_stream_index]; - - // Set up the codec to decode the audio stream - AVCodec const* codec = avcodec_find_decoder(m_audio_stream->codecpar->codec_id); - if (codec == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to find a suitable decoder" }; - - m_codec_context = avcodec_alloc_context3(codec); - if (m_codec_context == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to allocate the codec context" }; - - if (avcodec_parameters_to_context(m_codec_context, m_audio_stream->codecpar) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to copy codec parameters" }; - - m_codec_context->thread_count = AK::min(static_cast(Core::System::hardware_concurrency()), 4); - - if (avcodec_open2(m_codec_context, codec, nullptr) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to open input for decoding" }; - - double duration_in_seconds = m_audio_stream->duration * time_base(); - m_total_samples = AK::round_to(m_codec_context->sample_rate * duration_in_seconds); - - // Prepare packet and frame buffers - m_packet = av_packet_alloc(); - if (m_packet == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to allocate packet" }; - - m_frame = av_frame_alloc(); - if (m_frame == nullptr) - return LoaderError { LoaderError::Category::IO, "Failed to allocate frame" }; - - return {}; -} - -double OggLoaderPlugin::time_base() const -{ - return static_cast(m_audio_stream->time_base.num) / m_audio_stream->time_base.den; -} - -bool OggLoaderPlugin::sniff(SeekableStream& stream) -{ - LittleEndianInputBitStream bit_input { MaybeOwned(stream) }; - auto maybe_ogg = bit_input.read_bits(32); - return !maybe_ogg.is_error() && maybe_ogg.value() == 0x5367674F; // "OggS" -} - -static ErrorOr> extract_samples_from_frame(AVFrame& frame) -{ - size_t number_of_samples = frame.nb_samples; -#ifdef USE_FFMPEG_CH_LAYOUT - size_t number_of_channels = frame.ch_layout.nb_channels; -#else - size_t number_of_channels = frame.channels; -#endif - AVSampleFormat format = static_cast(frame.format); - - VERIFY(number_of_samples > 0); - - // FIXME: handle number_of_channels > 2 - if (number_of_channels != 1 && number_of_channels != 2) - return Error::from_string_view("Unsupported number of channels"sv); - - // FIXME: handle other formats - if (format != AV_SAMPLE_FMT_FLTP) - return Error::from_string_view("Unsupported sample format"sv); - - // FIXME: handle non-planar data (this is also implied by *P format(s) above) - if (av_sample_fmt_is_planar(format) != 1) - return Error::from_string_view("Non-planar sample data is not supported yet"sv); - - auto read_sample = [&](uint8_t* plane, size_t sample) -> float { - switch (format) { - case AV_SAMPLE_FMT_FLTP: - return reinterpret_cast(plane)[sample]; - default: - VERIFY_NOT_REACHED(); - } - }; - - auto samples = TRY(FixedArray::create(number_of_samples)); - for (size_t sample = 0; sample < number_of_samples; ++sample) { - if (number_of_channels == 1) { - samples.unchecked_at(sample) = Sample { read_sample(frame.extended_data[0], sample) }; - } else { - samples.unchecked_at(sample) = Sample { - read_sample(frame.extended_data[0], sample), - read_sample(frame.extended_data[1], sample), - }; - } - } - return samples; -} - -ErrorOr>, LoaderError> OggLoaderPlugin::load_chunks(size_t samples_to_read_from_input) -{ - Vector> chunks {}; - - for (;;) { - // Obtain a packet and send it to the decoder - if (av_read_frame(m_format_context, m_packet) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to read frame" }; - if (avcodec_send_packet(m_codec_context, m_packet) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to send packet" }; - av_packet_unref(m_packet); - - // Ask the decoder for a new frame. We might not have sent enough data yet - auto receive_frame_error = avcodec_receive_frame(m_codec_context, m_frame); - if (receive_frame_error == 0) { - chunks.append(TRY(extract_samples_from_frame(*m_frame))); - m_loaded_samples += m_frame->nb_samples; - - samples_to_read_from_input -= AK::min(samples_to_read_from_input, m_frame->nb_samples); - if (samples_to_read_from_input == 0) - break; - continue; - } - - if (receive_frame_error == AVERROR(EAGAIN)) - continue; - if (receive_frame_error == AVERROR_EOF) - return Error::from_errno(EOF); - - return LoaderError { LoaderError::Category::IO, "Failed to receive frame" }; - } - - av_frame_unref(m_frame); - - return chunks; -} - -MaybeLoaderError OggLoaderPlugin::reset() -{ - return seek(0); -} - -MaybeLoaderError OggLoaderPlugin::seek(int sample_index) -{ - auto sample_position_in_seconds = static_cast(sample_index) / m_codec_context->sample_rate; - auto sample_timestamp = AK::round_to(sample_position_in_seconds / time_base()); - - if (av_seek_frame(m_format_context, m_audio_stream->index, sample_timestamp, 0) < 0) - return LoaderError { LoaderError::Category::IO, "Failed to seek" }; - - m_loaded_samples = sample_index; - return {}; -} - -u32 OggLoaderPlugin::sample_rate() -{ - VERIFY(m_codec_context != nullptr); - return m_codec_context->sample_rate; -} - -u16 OggLoaderPlugin::num_channels() -{ - VERIFY(m_codec_context != nullptr); -#ifdef USE_FFMPEG_CH_LAYOUT - return m_codec_context->ch_layout.nb_channels; -#else - return m_codec_context->channels; -#endif -} - -PcmSampleFormat OggLoaderPlugin::pcm_format() -{ - // FIXME: pcm_format() is unused, always return Float for now - return PcmSampleFormat::Float32; -} - -} diff --git a/Userland/Libraries/LibMedia/CMakeLists.txt b/Userland/Libraries/LibMedia/CMakeLists.txt index 4e476b6c604..c4fa64050b6 100644 --- a/Userland/Libraries/LibMedia/CMakeLists.txt +++ b/Userland/Libraries/LibMedia/CMakeLists.txt @@ -30,7 +30,7 @@ endif() if (HAS_FFMPEG) list(APPEND SOURCES - Audio/OggLoader.cpp + Audio/FFmpegLoader.cpp FFmpeg/FFmpegVideoDecoder.cpp ) else() @@ -41,7 +41,7 @@ serenity_lib(LibMedia media) target_link_libraries(LibMedia PRIVATE LibCore LibCrypto LibRIFF LibIPC LibGfx LibThreading LibUnicode) if (HAS_FFMPEG) - target_link_libraries(LibMedia PRIVATE PkgConfig::AVCODEC PkgConfig::AVFORMAT) + target_link_libraries(LibMedia PRIVATE PkgConfig::AVCODEC PkgConfig::AVFORMAT PkgConfig::AVUTIL) endif() if (HAVE_PULSEAUDIO)