From 81001b37ceb708fec344e91073c10e02614d0355 Mon Sep 17 00:00:00 2001 From: Zaggy1024 Date: Wed, 19 Jun 2024 18:01:41 -0500 Subject: [PATCH] LibMedia: Use FFmpeg to decode more video formats VP9 continues to function, but this also allows AV1 to be decoded. With this commit, H.264 is still non-functional, as the decoder requires some extra initial data from the track definition in the Matroska file. --- .github/actions/setup/action.yml | 4 +- Documentation/BuildInstructionsLadybird.md | 6 +- Meta/CMake/lagom_compile_options.cmake | 4 + Userland/Libraries/LibMedia/CMakeLists.txt | 7 + .../Libraries/LibMedia/FFmpeg/FFmpegForward.h | 13 ++ .../Libraries/LibMedia/FFmpeg/FFmpegHelpers.h | 48 ++++ .../LibMedia/FFmpeg/FFmpegVideoDecoder.cpp | 216 ++++++++++++++++++ .../LibMedia/FFmpeg/FFmpegVideoDecoder.h | 33 +++ .../Libraries/LibMedia/PlaybackManager.cpp | 16 +- 9 files changed, 330 insertions(+), 17 deletions(-) create mode 100644 Userland/Libraries/LibMedia/FFmpeg/FFmpegForward.h create mode 100644 Userland/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h create mode 100644 Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp create mode 100644 Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 16a587d0f16..37503e015ea 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -25,7 +25,7 @@ runs: sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install autoconf autoconf-archive automake build-essential cmake fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev + sudo apt-get install autoconf autoconf-archive automake build-essential cmake libavcodec-dev fonts-liberation2 zip curl tar ccache clang-18 clang++-18 lld-18 gcc-13 g++-13 libstdc++-13-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev libpulse-dev libssl-dev libegl1-mesa-dev sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 @@ -51,7 +51,7 @@ runs: set -e sudo xcode-select --switch /Applications/Xcode_15.4.app brew update - brew install autoconf autoconf-archive automake coreutils bash ninja wabt ccache unzip qt llvm@18 + brew install autoconf autoconf-archive automake coreutils bash ffmpeg ninja wabt ccache unzip qt llvm@18 - name: 'Install vcpkg' shell: bash diff --git a/Documentation/BuildInstructionsLadybird.md b/Documentation/BuildInstructionsLadybird.md index db3e7b9690f..a241973131a 100644 --- a/Documentation/BuildInstructionsLadybird.md +++ b/Documentation/BuildInstructionsLadybird.md @@ -9,7 +9,7 @@ NOTE: In all of the below lists of packages, the Qt6 multimedia package is not n On Debian/Ubuntu required packages include, but are not limited to: ``` -sudo apt install autoconf autoconf-archive automake build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar +sudo apt install autoconf autoconf-archive automake build-essential cmake libavcodec-dev libgl1-mesa-dev ninja-build qt6-base-dev qt6-tools-dev-tools qt6-multimedia-dev ccache fonts-liberation2 zip unzip curl tar ``` For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available: @@ -21,7 +21,7 @@ sudo apt install qt6-wayland On Arch Linux/Manjaro: ``` -sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive +sudo pacman -S --needed base-devel cmake ffmpeg libgl ninja qt6-base qt6-tools qt6-wayland qt6-multimedia ccache ttf-liberation curl unzip zip tar autoconf-archive ``` On Fedora or derivatives: @@ -56,7 +56,7 @@ Xcode 14 versions before 14.3 might crash while building ladybird. Xcode 14.3 or ``` xcode-select --install -brew install autoconf autoconf-archive automake cmake ninja ccache pkg-config +brew install autoconf autoconf-archive automake cmake ffmpeg ninja ccache pkg-config ``` If you also plan to use the Qt chrome on macOS: diff --git a/Meta/CMake/lagom_compile_options.cmake b/Meta/CMake/lagom_compile_options.cmake index 6a5f67bd237..8b3c0919eb2 100644 --- a/Meta/CMake/lagom_compile_options.cmake +++ b/Meta/CMake/lagom_compile_options.cmake @@ -21,6 +21,10 @@ if (LINUX) add_compile_options(-D_FILE_OFFSET_BITS=64) endif() +if (APPLE) + list(APPEND CMAKE_PREFIX_PATH /opt/homebrew) +endif() + if (CMAKE_BUILD_TYPE STREQUAL "Debug") if (NOT MSVC) add_compile_options(-ggdb3) diff --git a/Userland/Libraries/LibMedia/CMakeLists.txt b/Userland/Libraries/LibMedia/CMakeLists.txt index 1110b9c7226..523813e93af 100644 --- a/Userland/Libraries/LibMedia/CMakeLists.txt +++ b/Userland/Libraries/LibMedia/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES Color/TransferCharacteristics.cpp Containers/Matroska/MatroskaDemuxer.cpp Containers/Matroska/Reader.cpp + FFmpeg/FFmpegVideoDecoder.cpp PlaybackManager.cpp VideoFrame.cpp Video/VP9/Decoder.cpp @@ -15,3 +16,9 @@ set(SOURCES serenity_lib(LibMedia media) target_link_libraries(LibMedia PRIVATE LibCore LibIPC LibGfx LibThreading) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec) +target_include_directories(LibMedia PRIVATE ${AVCODEC_INCLUDE_DIRS}) +target_link_directories(LibMedia PRIVATE ${AVCODEC_LIBRARY_DIRS}) +target_link_libraries(LibMedia PRIVATE ${AVCODEC_LIBRARIES}) diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegForward.h b/Userland/Libraries/LibMedia/FFmpeg/FFmpegForward.h new file mode 100644 index 00000000000..3c59481b06f --- /dev/null +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegForward.h @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2024, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +extern "C" { +struct AVCodecContext; +struct AVPacket; +struct AVFrame; +} diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h b/Userland/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h new file mode 100644 index 00000000000..74ebc5cb9a6 --- /dev/null +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegHelpers.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +extern "C" { +#include +} + +namespace Media::FFmpeg { + +static inline AVCodecID ffmpeg_codec_id_from_serenity_codec_id(CodecID codec) +{ + switch (codec) { + case CodecID::VP8: + return AV_CODEC_ID_VP8; + case CodecID::VP9: + return AV_CODEC_ID_VP9; + case CodecID::H261: + return AV_CODEC_ID_H261; + case CodecID::MPEG1: + case CodecID::H262: + return AV_CODEC_ID_MPEG2VIDEO; + case CodecID::H263: + return AV_CODEC_ID_H263; + case CodecID::H264: + return AV_CODEC_ID_H264; + case CodecID::H265: + return AV_CODEC_ID_HEVC; + case CodecID::AV1: + return AV_CODEC_ID_AV1; + case CodecID::Theora: + return AV_CODEC_ID_THEORA; + case CodecID::Vorbis: + return AV_CODEC_ID_VORBIS; + case CodecID::Opus: + return AV_CODEC_ID_OPUS; + default: + return AV_CODEC_ID_NONE; + } +} + +} diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp new file mode 100644 index 00000000000..601135e5791 --- /dev/null +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2024, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include "FFmpegHelpers.h" +#include "FFmpegVideoDecoder.h" + +namespace Media::FFmpeg { + +static AVPixelFormat negotiate_output_format(AVCodecContext*, AVPixelFormat const* formats) +{ + while (*formats >= 0) { + switch (*formats) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV420P10: + case AV_PIX_FMT_YUV420P12: + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV422P10: + case AV_PIX_FMT_YUV422P12: + case AV_PIX_FMT_YUV444P: + case AV_PIX_FMT_YUV444P10: + case AV_PIX_FMT_YUV444P12: + return *formats; + default: + break; + } + formats++; + } + return AV_PIX_FMT_NONE; +} + +DecoderErrorOr> FFmpegVideoDecoder::try_create(CodecID codec_id) +{ + AVCodecContext* codec_context = nullptr; + AVPacket* packet = nullptr; + AVFrame* frame = nullptr; + ArmedScopeGuard memory_guard { + [&] { + avcodec_free_context(&codec_context); + av_packet_free(&packet); + av_frame_free(&frame); + } + }; + + auto ff_codec_id = ffmpeg_codec_id_from_serenity_codec_id(codec_id); + auto const* codec = avcodec_find_decoder(ff_codec_id); + if (!codec) + return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id); + + codec_context = avcodec_alloc_context3(codec); + if (!codec_context) + return DecoderError::format(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg codec context for codec {}", codec_id); + + codec_context->get_format = negotiate_output_format; + + codec_context->thread_count = static_cast(min(Core::System::hardware_concurrency(), 4)); + + if (avcodec_open2(codec_context, codec, nullptr) < 0) + return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id); + + packet = av_packet_alloc(); + if (!packet) + return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg packet"sv); + + frame = av_frame_alloc(); + if (!frame) + return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg frame"sv); + + memory_guard.disarm(); + return DECODER_TRY_ALLOC(try_make(codec_context, packet, frame)); +} + +FFmpegVideoDecoder::FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame) + : m_codec_context(codec_context) + , m_packet(packet) + , m_frame(frame) +{ +} + +FFmpegVideoDecoder::~FFmpegVideoDecoder() +{ + av_packet_free(&m_packet); + av_frame_free(&m_frame); + avcodec_free_context(&m_codec_context); +} + +DecoderErrorOr FFmpegVideoDecoder::receive_sample(ReadonlyBytes sample) +{ + VERIFY(sample.size() < NumericLimits::max()); + + m_packet->data = const_cast(sample.data()); + m_packet->size = static_cast(sample.size()); + + auto result = avcodec_send_packet(m_codec_context, m_packet); + switch (result) { + case 0: + return {}; + case AVERROR(EAGAIN): + return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder cannot decode any more data until frames have been retrieved"sv); + case AVERROR_EOF: + return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv); + case AVERROR(EINVAL): + return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv); + case AVERROR(ENOMEM): + return DecoderError::with_description(DecoderErrorCategory::Memory, "FFmpeg codec ran out of internal memory"sv); + default: + return DecoderError::with_description(DecoderErrorCategory::Corrupted, "FFmpeg codec reports that the data is corrupted"sv); + } +} + +DecoderErrorOr> FFmpegVideoDecoder::get_decoded_frame() +{ + auto result = avcodec_receive_frame(m_codec_context, m_frame); + + switch (result) { + case 0: { + auto color_primaries = static_cast(m_frame->color_primaries); + auto transfer_characteristics = static_cast(m_frame->color_trc); + auto matrix_coefficients = static_cast(m_frame->colorspace); + auto color_range = [&] { + switch (m_frame->color_range) { + case AVColorRange::AVCOL_RANGE_MPEG: + return VideoFullRangeFlag::Studio; + case AVColorRange::AVCOL_RANGE_JPEG: + return VideoFullRangeFlag::Full; + default: + return VideoFullRangeFlag::Unspecified; + } + }(); + auto cicp = CodingIndependentCodePoints { color_primaries, transfer_characteristics, matrix_coefficients, color_range }; + + size_t bit_depth = [&] { + switch (m_frame->format) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV444P: + return 8; + case AV_PIX_FMT_YUV420P10: + case AV_PIX_FMT_YUV422P10: + case AV_PIX_FMT_YUV444P10: + return 10; + case AV_PIX_FMT_YUV420P12: + case AV_PIX_FMT_YUV422P12: + case AV_PIX_FMT_YUV444P12: + return 12; + } + VERIFY_NOT_REACHED(); + }(); + size_t component_size = (bit_depth + 7) / 8; + + auto subsampling = [&]() -> Subsampling { + switch (m_frame->format) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUV420P10: + case AV_PIX_FMT_YUV420P12: + return { true, true }; + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUV422P10: + case AV_PIX_FMT_YUV422P12: + return { true, false }; + case AV_PIX_FMT_YUV444P: + case AV_PIX_FMT_YUV444P10: + case AV_PIX_FMT_YUV444P12: + return { false, false }; + + default: + VERIFY_NOT_REACHED(); + } + }(); + + auto size = Gfx::Size { m_frame->width, m_frame->height }; + + auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(size, bit_depth, cicp, subsampling)); + + for (u32 plane = 0; plane < 3; plane++) { + VERIFY(m_frame->linesize[plane] != 0); + if (m_frame->linesize[plane] < 0) + return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "Reversed scanlines are not supported"sv); + + bool const use_subsampling = plane > 0; + auto plane_size = (use_subsampling ? subsampling.subsampled_size(size) : size).to_type(); + + auto output_line_size = plane_size.width() * component_size; + VERIFY(output_line_size <= static_cast(m_frame->linesize[plane])); + + auto const* source = m_frame->data[plane]; + VERIFY(source != nullptr); + auto* destination = frame->get_raw_plane_data(plane); + VERIFY(destination != nullptr); + + for (size_t row = 0; row < plane_size.height(); row++) { + memcpy(destination, source, output_line_size); + source += m_frame->linesize[plane]; + destination += output_line_size; + } + } + + return frame; + } + case AVERROR(EAGAIN): + return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder has no frames available, send more input"sv); + case AVERROR_EOF: + return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv); + case AVERROR(EINVAL): + return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv); + default: + return DecoderError::format(DecoderErrorCategory::Unknown, "FFmpeg codec encountered an unexpected error retreiving frames with code {:x}", result); + } +} + +} diff --git a/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h new file mode 100644 index 00000000000..be1baa283e1 --- /dev/null +++ b/Userland/Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +#include "FFmpegForward.h" + +namespace Media::FFmpeg { + +class FFmpegVideoDecoder final : public VideoDecoder { +public: + static DecoderErrorOr> try_create(CodecID); + FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame); + ~FFmpegVideoDecoder(); + + DecoderErrorOr receive_sample(ReadonlyBytes sample) override; + DecoderErrorOr> get_decoded_frame() override; + +private: + DecoderErrorOr decode_single_sample(Duration timestamp, u8* data, int size); + + AVCodecContext* m_codec_context; + AVPacket* m_packet; + AVFrame* m_frame; +}; + +} diff --git a/Userland/Libraries/LibMedia/PlaybackManager.cpp b/Userland/Libraries/LibMedia/PlaybackManager.cpp index 2733d0cd394..93ee932788e 100644 --- a/Userland/Libraries/LibMedia/PlaybackManager.cpp +++ b/Userland/Libraries/LibMedia/PlaybackManager.cpp @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include "PlaybackManager.h" @@ -700,18 +701,9 @@ DecoderErrorOr> PlaybackManager::create(NonnullOw dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier()); auto codec_id = TRY(demuxer->get_codec_id_for_track(track)); - OwnPtr decoder; - switch (codec_id) { - case CodecID::VP9: - decoder = DECODER_TRY_ALLOC(try_make()); - break; - - default: - return DecoderError::format(DecoderErrorCategory::Invalid, "Unsupported codec: {}", codec_id); - } - auto decoder_non_null = decoder.release_nonnull(); + NonnullOwnPtr decoder = TRY(FFmpeg::FFmpegVideoDecoder::try_create(codec_id)); auto frame_queue = DECODER_TRY_ALLOC(VideoFrameQueue::create()); - auto playback_manager = DECODER_TRY_ALLOC(try_make(demuxer, track, move(decoder_non_null), move(frame_queue))); + auto playback_manager = DECODER_TRY_ALLOC(try_make(demuxer, track, move(decoder), move(frame_queue))); playback_manager->m_state_update_timer = Core::Timer::create_single_shot(0, [&self = *playback_manager] { self.timer_callback(); });